feat: multi-key round-robin per provider (C5)

.env.example

@@ -4,6 +4,9 @@
 OPENAI_API_KEY=
 ANTHROPIC_API_KEY=
 GEMINI_API_KEY=
+# Optional extra key(s) for a provider — Sentinel round-robins across a provider's key pool
+# (configured via "apiKeyEnvs" in sentinel.config.json) to spread load across free-tier limits.
+GEMINI_API_KEY_2=
 GROQ_API_KEY=
 
 # ── Sentinel ─────────────────────────────────────────────────────────

CHANGELOG.md

@@ -10,6 +10,7 @@ All notable changes to Sentinel are documented here. The format follows
 
 - Per-request **cost tracking**: set a `pricing` map (USD per 1K tokens, per model) in `sentinel.config.json` and every trace records a `costUsd`; the dashboard shows total spend, spend saved by the cache, and cost over time.
 - **Native Anthropic provider**: set `"type": "anthropic"` on a provider and Sentinel speaks Anthropic's Messages API (`x-api-key`, request/response/stream translation) while clients keep using the OpenAI shape.
+- **Multi-key round-robin**: a provider can list `apiKeyEnvs` (a key pool); Sentinel rotates across the keys per request to spread load and survive free-tier rate limits.
 
 ### Fixed
 

README.md

@@ -59,7 +59,7 @@ Two git-ignored files:
 - **`.env`** — secrets only: provider API keys, and the `SENTINEL_API_KEYS` clients use to authenticate to Sentinel. See [`.env.example`](./.env.example).
 - **`sentinel.config.json`** — which providers exist and which model routes where. See [`sentinel.config.example.json`](./sentinel.config.example.json).
 
-Each provider sets a `baseUrl` (or `baseUrlEnv` to read it from the environment) and an optional `apiKeyEnv` naming the env var that holds its key. The `models` map routes a model name to a provider; `defaultProvider` handles anything not listed.
+Each provider sets a `baseUrl` (or `baseUrlEnv` to read it from the environment) and an optional `apiKeyEnv` naming the env var that holds its key — or `apiKeyEnvs` (an array) to give a provider a **pool of keys Sentinel round-robins across**, spreading load to survive free-tier limits. The `models` map routes a model name to a provider; `defaultProvider` handles anything not listed.
 
 ```json
 {

SECURITY_REVIEW_LOG.md

@@ -77,4 +77,6 @@ Each item: assume an attacker is actively trying it. Tick only when a test prove
 
 | SR-007 | 2026-06-30 | 1, 4 | PR3 native Anthropic adapter | A second outbound auth path (`x-api-key`) and a new response/stream shape: risk of leaking the Anthropic key to logs, following a malicious redirect, or trusting a malformed upstream body | high | mitigated | The Anthropic key is sent only in the outbound `x-api-key` header — never logged (`logRedaction` already redacts inbound `x-api-key`; Fastify omits request headers by default) and never returned to clients; it comes from `ANTHROPIC_API_KEY` via `apiKeyEnv`, never hard-coded. The adapter reuses `redirect: 'error'`, so a malicious provider can't 3xx the prompt to another host (tested), and the base URL still comes only from the config allow-list, never request input. The Anthropic response is parsed through a Zod schema before translation; an unexpected shape ⇒ `UpstreamError` (502), never a malformed pass-through (tested). |
 
+| SR-008 | 2026-06-30 | 1 | PR4 multi-key round-robin | Holding several provider keys per provider widens the secret surface — a key could leak via logs/errors, or a blank slot send an empty key | low | mitigated | Keys come only from env (`apiKeyEnv`/`apiKeyEnvs`), never the repo or request input; each is sent only in the outbound auth header (`Authorization`/`x-api-key`), redacted in logs and never returned to clients. Rotation is an in-memory counter over the resolved pool — no provider key is persisted or traced (traces still store only the SHA-256 of the *Sentinel client* key). Unset/empty env vars are skipped, so a misconfigured slot can't send a blank key (tested). |
+
 > Add a row per high-risk change. Status ∈ {open, mitigated, accepted}. Severity ∈ {low, med, high, critical}.

packages/gateway/src/config.test.ts

@@ -89,8 +89,8 @@ describe('loadConfig', () => {
   it('resolves providers, base URLs, and keys', () => {
     const cfg = loadConfig({ path: 'x', env, readFile: () => validConfig });
     expect(cfg.providers.get('ollama')?.baseUrl).toBe('http://localhost:11434/v1');
-    expect(cfg.providers.get('ollama')?.apiKey).toBeUndefined();
-    expect(cfg.providers.get('openai')?.apiKey).toBe('sk-test');
+    expect(cfg.providers.get('ollama')?.apiKeys).toEqual([]);
+    expect(cfg.providers.get('openai')?.apiKeys).toEqual(['sk-test']);
     expect(cfg.models.get('gpt-4o-mini')).toBe('openai');
     expect(cfg.defaultProvider).toBe('ollama');
   });
@@ -132,7 +132,27 @@ describe('loadConfig', () => {
       readFile: () => cfg,
     });
     expect(resolved.providers.get('claude')?.type).toBe('anthropic');
-    expect(resolved.providers.get('claude')?.apiKey).toBe('sk-ant');
+    expect(resolved.providers.get('claude')?.apiKeys).toEqual(['sk-ant']);
+  });
+
+  it('collects a round-robin key pool from apiKeyEnvs (+ apiKeyEnv), skipping unset keys', () => {
+    const cfg = JSON.stringify({
+      providers: {
+        groq: {
+          type: 'openai-compatible',
+          baseUrl: 'https://g',
+          apiKeyEnv: 'G1',
+          apiKeyEnvs: ['G2', 'G3'],
+        },
+      },
+      models: { m: 'groq' },
+    });
+    const resolved = loadConfig({
+      path: 'x',
+      env: { G1: 'k1', G2: 'k2', G3: '' }, // G3 empty → skipped
+      readFile: () => cfg,
+    });
+    expect(resolved.providers.get('groq')?.apiKeys).toEqual(['k1', 'k2']);
   });
 
   it('rejects an unknown provider adapter type', () => {

packages/gateway/src/config.ts

@@ -113,6 +113,8 @@ const providerConfigSchema = z
     baseUrl: z.string().url().optional(),
     baseUrlEnv: z.string().min(1).optional(),
     apiKeyEnv: z.string().min(1).optional(),
+    /** Multiple key env vars to round-robin across (combined with `apiKeyEnv` if both are set). */
+    apiKeyEnvs: z.array(z.string().min(1)).optional(),
     rpm: z.coerce.number().int().nonnegative().optional(),
   })
   .refine((p) => (p.baseUrl === undefined) !== (p.baseUrlEnv === undefined), {
@@ -168,7 +170,8 @@ export interface ResolvedProvider {
   /** Which adapter serves this provider: a generic OpenAI-compatible HTTP API or Anthropic's Messages API. */
   type: 'openai-compatible' | 'anthropic';
   baseUrl: string;
-  apiKey: string | undefined;
+  /** Resolved API keys for this provider (0 = keyless, >1 = round-robin pool). */
+  apiKeys: string[];
   /** Per-provider requests-per-minute limit for the throttle (omitted = unlimited). */
   rpm?: number;
 }
@@ -230,12 +233,12 @@ export function loadConfig(options: LoadConfigOptions): ResolvedConfig {
   const providers = new Map<string, ResolvedProvider>();
   for (const [name, p] of Object.entries(parsed.data.providers)) {
     const baseUrl = p.baseUrl ?? readEnvOrThrow(options.env, p.baseUrlEnv, name);
-    const apiKey = p.apiKeyEnv === undefined ? undefined : options.env[p.apiKeyEnv];
+    const apiKeys = resolveApiKeys(options.env, p.apiKeyEnv, p.apiKeyEnvs);
     providers.set(name, {
       name,
       type: p.type,
       baseUrl,
-      apiKey,
+      apiKeys,
       ...(p.rpm !== undefined ? { rpm: p.rpm } : {}),
     });
   }
@@ -250,6 +253,21 @@ export function loadConfig(options: LoadConfigOptions): ResolvedConfig {
   };
 }
 
+/** Resolves a provider's key pool from `apiKeyEnv` (single) + `apiKeyEnvs` (many); skips unset/empty. */
+function resolveApiKeys(
+  env: NodeJS.ProcessEnv,
+  single: string | undefined,
+  multiple: string[] | undefined,
+): string[] {
+  const names = [...(single !== undefined ? [single] : []), ...(multiple ?? [])];
+  const keys: string[] = [];
+  for (const varName of names) {
+    const value = env[varName];
+    if (value !== undefined && value.length > 0) keys.push(value);
+  }
+  return keys;
+}
+
 function readEnvOrThrow(env: NodeJS.ProcessEnv, key: string | undefined, provider: string): string {
   if (key === undefined) {
     throw new ConfigError(`provider "${provider}" is missing a base URL`);

packages/gateway/src/providers/anthropic.ts

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 import type { ChatCompletionRequest } from '../schemas.js';
-import type { FetchLike, Provider } from './types.js';
+import type { ApiKeySource, FetchLike, Provider } from './types.js';
 import { UpstreamError } from '../errors.js';
 
 /**
@@ -18,7 +18,8 @@ const DEFAULT_MAX_TOKENS = 4096;
 export interface AnthropicOptions {
   name: string;
   baseUrl: string;
-  apiKey?: string | undefined;
+  /** A fixed key, or a supplier that returns the next key (round-robin across a pool). */
+  apiKey?: ApiKeySource | undefined;
   /** Injectable fetch (defaults to global `fetch`); handy for tests. */
   fetchImpl?: FetchLike;
 }
@@ -50,8 +51,9 @@ export function createAnthropicProvider(options: AnthropicOptions): Provider {
       'content-type': 'application/json',
       'anthropic-version': ANTHROPIC_VERSION,
     };
-    if (options.apiKey !== undefined && options.apiKey.length > 0) {
-      headers['x-api-key'] = options.apiKey;
+    const key = typeof options.apiKey === 'function' ? options.apiKey() : options.apiKey;
+    if (key !== undefined && key.length > 0) {
+      headers['x-api-key'] = key;
     }
     return headers;
   }

packages/gateway/src/providers/openai-compatible.ts

@@ -1,11 +1,12 @@
 import type { ChatCompletionRequest } from '../schemas.js';
-import type { FetchLike, Provider } from './types.js';
+import type { ApiKeySource, FetchLike, Provider } from './types.js';
 import { UpstreamError } from '../errors.js';
 
 export interface OpenAICompatibleOptions {
   name: string;
   baseUrl: string;
-  apiKey?: string | undefined;
+  /** A fixed key, or a supplier that returns the next key (round-robin across a pool). */
+  apiKey?: ApiKeySource | undefined;
   /** Injectable fetch (defaults to global `fetch`); handy for tests. */
   fetchImpl?: FetchLike;
 }
@@ -20,8 +21,9 @@ export function createOpenAICompatibleProvider(options: OpenAICompatibleOptions)
 
   function buildHeaders(): Record<string, string> {
     const headers: Record<string, string> = { 'content-type': 'application/json' };
-    if (options.apiKey !== undefined && options.apiKey.length > 0) {
-      headers.authorization = `Bearer ${options.apiKey}`;
+    const key = typeof options.apiKey === 'function' ? options.apiKey() : options.apiKey;
+    if (key !== undefined && key.length > 0) {
+      headers.authorization = `Bearer ${key}`;
     }
     return headers;
   }

packages/gateway/src/providers/registry.test.ts

@@ -11,7 +11,7 @@ const config: ResolvedConfig = {
         name: 'ollama',
         type: 'openai-compatible',
         baseUrl: 'http://localhost:11434/v1',
-        apiKey: undefined,
+        apiKeys: [],
       },
     ],
     [
@@ -20,7 +20,7 @@ const config: ResolvedConfig = {
         name: 'openai',
         type: 'openai-compatible',
         baseUrl: 'https://api.openai.com/v1',
-        apiKey: 'k',
+        apiKeys: ['k'],
       },
     ],
   ]),
@@ -67,7 +67,10 @@ describe('createRegistry', () => {
   it('builds the Anthropic adapter for an anthropic-typed provider', async () => {
     const anthropicConfig: ResolvedConfig = {
       providers: new Map([
-        ['claude', { name: 'claude', type: 'anthropic', baseUrl: 'http://h/v1', apiKey: 'sk-ant' }],
+        [
+          'claude',
+          { name: 'claude', type: 'anthropic', baseUrl: 'http://h/v1', apiKeys: ['sk-ant'] },
+        ],
       ]),
       models: new Map([['claude-3-5-sonnet', 'claude']]),
       defaultProvider: undefined,
@@ -89,4 +92,53 @@ describe('createRegistry', () => {
     expect(call[0]).toContain('/messages');
     expect(call[1].headers['x-api-key']).toBe('sk-ant');
   });
+
+  it('round-robins across a provider key pool on successive requests', async () => {
+    const pooled: ResolvedConfig = {
+      providers: new Map([
+        [
+          'groq',
+          {
+            name: 'groq',
+            type: 'openai-compatible',
+            baseUrl: 'http://h/v1',
+            apiKeys: ['k1', 'k2'],
+          },
+        ],
+      ]),
+      models: new Map([['m', 'groq']]),
+      defaultProvider: undefined,
+      pricing: new Map(),
+    };
+    const seen: (string | undefined)[] = [];
+    const fetchImpl = vi.fn(async (_url: string, init: { headers: Record<string, string> }) => {
+      seen.push(init.headers.authorization);
+      return Promise.resolve(new Response('{}', { status: 200 }));
+    });
+    const registry = createRegistry(pooled, { fetchImpl });
+    for (let i = 0; i < 3; i++) {
+      await registry.resolve('m').chat({ model: 'm', messages: [{ role: 'user', content: 'hi' }] });
+    }
+    expect(seen).toEqual(['Bearer k1', 'Bearer k2', 'Bearer k1']);
+  });
+
+  it('uses no auth header for a keyless provider pool', async () => {
+    const keyless: ResolvedConfig = {
+      providers: new Map([
+        [
+          'ollama',
+          { name: 'ollama', type: 'openai-compatible', baseUrl: 'http://h/v1', apiKeys: [] },
+        ],
+      ]),
+      models: new Map([['m', 'ollama']]),
+      defaultProvider: undefined,
+      pricing: new Map(),
+    };
+    const fetchImpl = vi.fn(async (_url: string, _init: { headers: Record<string, string> }) =>
+      Promise.resolve(new Response('{}', { status: 200 })),
+    );
+    const registry = createRegistry(keyless, { fetchImpl });
+    await registry.resolve('m').chat({ model: 'm', messages: [{ role: 'user', content: 'hi' }] });
+    expect(fetchImpl.mock.calls[0]![1].headers.authorization).toBeUndefined();
+  });
 });

packages/gateway/src/providers/registry.ts

@@ -1,9 +1,21 @@
 import type { ResolvedConfig } from '../config.js';
-import type { FetchLike, Provider } from './types.js';
+import type { ApiKeySource, FetchLike, Provider } from './types.js';
 import { createOpenAICompatibleProvider } from './openai-compatible.js';
 import { createAnthropicProvider } from './anthropic.js';
 import { ModelNotFoundError } from '../errors.js';
 
+/** A key source for a provider: undefined (keyless), the single key, or a round-robin over the pool. */
+function keySupplier(keys: readonly string[]): ApiKeySource | undefined {
+  if (keys.length === 0) return undefined;
+  if (keys.length === 1) return keys[0];
+  let index = 0;
+  return () => {
+    const key = keys[index % keys.length];
+    index += 1;
+    return key;
+  };
+}
+
 export interface ProviderRegistry {
   /** Resolves a model name to the provider that serves it. */
   resolve(model: string): Provider;
@@ -20,10 +32,11 @@ export function createRegistry(
 ): ProviderRegistry {
   const providers = new Map<string, Provider>();
   for (const resolved of config.providers.values()) {
+    const apiKey = keySupplier(resolved.apiKeys);
     const adapterOptions = {
       name: resolved.name,
       baseUrl: resolved.baseUrl,
-      apiKey: resolved.apiKey,
+      ...(apiKey !== undefined ? { apiKey } : {}),
       ...(options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}),
     };
     providers.set(

packages/gateway/src/providers/types.ts

@@ -12,6 +12,12 @@ export type FetchLike = (
   },
 ) => Promise<Response>;
 
+/**
+ * A provider API key: either a fixed string, or a supplier that returns the next
+ * key to use (so the registry can round-robin across a multi-key pool per call).
+ */
+export type ApiKeySource = string | (() => string | undefined);
+
 /** A provider Sentinel can forward chat-completion requests to. */
 export interface Provider {
   readonly name: string;

sentinel.config.example.json

@@ -13,7 +13,7 @@
     "gemini": {
       "type": "openai-compatible",
       "baseUrl": "https://generativelanguage.googleapis.com/v1beta/openai",
-      "apiKeyEnv": "GEMINI_API_KEY"
+      "apiKeyEnvs": ["GEMINI_API_KEY", "GEMINI_API_KEY_2"]
     },
     "groq": {
       "type": "openai-compatible",