From b3295e6fad046e3d299e77f42fe63211b16ee46f Mon Sep 17 00:00:00 2001 From: KochC Date: Fri, 27 Mar 2026 15:44:50 +0100 Subject: [PATCH 01/11] feat: expand tests, add ESLint, document env vars - Add 17 new integration tests: CORS edge cases (disallowed origins, no-origin header, OPTIONS for disallowed origin), auth (401/pass-through), and error handling (400/502/404) for /v1/chat/completions Closes #14, closes #16 - Add ESLint with flat config, npm run lint script, and Lint job in CI Closes #15 - Improve README with quickstart section, npm install instructions, and corrected package name; add type column to env vars table Closes #17 --- .github/workflows/ci.yml | 22 + .npmignore | 2 + README.md | 82 +++- eslint.config.js | 48 ++ index.test.js | 214 ++++++++- package-lock.json | 933 +++++++++++++++++++++++++++++++++++++++ package.json | 5 + 7 files changed, 1285 insertions(+), 21 deletions(-) create mode 100644 eslint.config.js create mode 100644 package-lock.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c072617..65673fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,25 @@ on: - dev jobs: + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Install dev dependencies + run: npm install + + - name: Run lint + run: npm run lint + test: name: Test (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest @@ -28,5 +47,8 @@ jobs: with: node-version: ${{ matrix.node-version }} + - name: Install dev dependencies + run: npm install + - name: Run tests run: npm test diff --git a/.npmignore b/.npmignore index 2f9c7a0..c1c7e82 100644 --- a/.npmignore +++ b/.npmignore @@ -3,4 +3,6 @@ index.test.js .release-please-manifest.json release-please-config.json CHANGELOG.md +eslint.config.js +node_modules/ diff --git a/README.md b/README.md index c7fe331..90a676b 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,62 @@ -# opencode-openai-proxy +# opencode-llm-proxy An [OpenCode](https://opencode.ai) plugin that starts a local OpenAI-compatible HTTP server backed by your OpenCode providers. -Any tool or application that speaks the OpenAI Chat Completions or Responses API can use it — including the Agile-V Studio platform, LangChain, custom scripts, etc. +Any tool or application that speaks the OpenAI Chat Completions or Responses API can use it — including LangChain, custom scripts, local frontends, etc. + +## Quickstart + +```bash +# 1. Install the npm package +npm install opencode-llm-proxy + +# 2. Register the plugin in your opencode.json +# (or use one of the manual install methods below) +``` + +Add to `opencode.json`: + +```json +{ + "plugin": ["opencode-llm-proxy"] +} +``` + +Then start OpenCode — the proxy starts automatically: + +```bash +opencode +# Proxy is now listening on http://127.0.0.1:4010 +``` + +Send a request: + +```bash +curl http://127.0.0.1:4010/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "github-copilot/claude-sonnet-4.6", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` ## Install -### As a global OpenCode plugin (recommended) +### As an npm plugin (recommended) + +```bash +npm install opencode-llm-proxy +``` + +Add to `opencode.json`: + +```json +{ + "plugin": ["opencode-llm-proxy"] +} +``` + +### As a global OpenCode plugin Copy `index.js` to your global plugin directory: @@ -24,16 +74,6 @@ Copy `index.js` to your project's plugin directory: cp index.js .opencode/plugins/openai-proxy.js ``` -### As an npm plugin - -Add it to your `opencode.json`: - -```json -{ - "plugin": ["opencode-openai-proxy"] -} -``` - ## Usage Start OpenCode normally. The proxy server starts automatically in the background: @@ -80,14 +120,16 @@ curl http://127.0.0.1:4010/v1/responses \ ## Configuration -| Environment variable | Default | Description | -|---|---|---| -| `OPENCODE_LLM_PROXY_HOST` | `127.0.0.1` | Bind host. Set to `0.0.0.0` to expose on LAN. | -| `OPENCODE_LLM_PROXY_PORT` | `4010` | Bind port. | -| `OPENCODE_LLM_PROXY_TOKEN` | _(none)_ | Optional bearer token. If set, all requests must include `Authorization: Bearer `. | -| `OPENCODE_LLM_PROXY_CORS_ORIGIN` | `*` | CORS `Access-Control-Allow-Origin` header value. Use a specific origin if browser clients send credentials. | +All configuration is done through environment variables. No configuration file is needed. + +| Variable | Type | Default | Description | +|---|---|---|---| +| `OPENCODE_LLM_PROXY_HOST` | string | `127.0.0.1` | Bind address. Set to `0.0.0.0` to expose on LAN. | +| `OPENCODE_LLM_PROXY_PORT` | integer | `4010` | TCP port the proxy listens on. | +| `OPENCODE_LLM_PROXY_TOKEN` | string | _(unset)_ | Optional bearer token. When set, every request must include `Authorization: Bearer `. Unset means no authentication required. | +| `OPENCODE_LLM_PROXY_CORS_ORIGIN` | string | `*` | Value of the `Access-Control-Allow-Origin` response header. Use a specific origin (e.g. `https://app.example.com`) when browser clients send credentials. | -The proxy answers browser preflight requests and adds CORS headers on success and error responses for `/health`, `/v1/models`, `/v1/chat/completions`, and `/v1/responses`. +The proxy adds CORS headers to all responses and handles `OPTIONS` preflight requests automatically. ### LAN example diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..45f5a7e --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,48 @@ +import js from "@eslint/js" + +export default [ + js.configs.recommended, + { + languageOptions: { + ecmaVersion: 2022, + sourceType: "module", + globals: { + // Node.js globals + process: "readonly", + globalThis: "readonly", + crypto: "readonly", + // Bun globals (used in OpenAIProxyPlugin) + Bun: "readonly", + // Web API globals available in both Node and Bun + Request: "readonly", + Response: "readonly", + URL: "readonly", + }, + }, + rules: { + "no-unused-vars": ["error", { argsIgnorePattern: "^_", varsIgnorePattern: "^_" }], + "no-console": "warn", + }, + }, + { + // Relax rules for the test file + files: ["*.test.js"], + languageOptions: { + globals: { + // node:test globals + describe: "readonly", + it: "readonly", + before: "readonly", + after: "readonly", + beforeEach: "readonly", + afterEach: "readonly", + }, + }, + rules: { + "no-unused-vars": "off", + }, + }, + { + ignores: ["node_modules/"], + }, +] diff --git a/index.test.js b/index.test.js index e381807..e4241f7 100644 --- a/index.test.js +++ b/index.test.js @@ -92,8 +92,220 @@ test("configured origin is returned for normal requests", async () => { } }) +test("disallowed origin does not receive its own origin back", async () => { + process.env.OPENCODE_LLM_PROXY_CORS_ORIGIN = "https://allowed.example.com" + + try { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/health", { + headers: { Origin: "https://evil.example.com" }, + }) + + const response = await handler(request) + + // The header must be the configured origin, not the request's origin + assert.equal(response.headers.get("access-control-allow-origin"), "https://allowed.example.com") + assert.notEqual(response.headers.get("access-control-allow-origin"), "https://evil.example.com") + } finally { + delete process.env.OPENCODE_LLM_PROXY_CORS_ORIGIN + } +}) + +test("request with no Origin header is handled gracefully", async () => { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/health") + + const response = await handler(request) + + assert.equal(response.status, 200) + // CORS header is still present (wildcard default) even without an Origin + assert.equal(response.headers.get("access-control-allow-origin"), "*") +}) + +test("OPTIONS preflight for disallowed origin returns configured origin, not request origin", async () => { + process.env.OPENCODE_LLM_PROXY_CORS_ORIGIN = "https://allowed.example.com" + + try { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "OPTIONS", + headers: { + Origin: "https://evil.example.com", + "Access-Control-Request-Method": "POST", + }, + }) + + const response = await handler(request) + + assert.equal(response.status, 204) + assert.equal(response.headers.get("access-control-allow-origin"), "https://allowed.example.com") + assert.notEqual(response.headers.get("access-control-allow-origin"), "https://evil.example.com") + } finally { + delete process.env.OPENCODE_LLM_PROXY_CORS_ORIGIN + } +}) + +// --------------------------------------------------------------------------- +// Integration: authentication +// --------------------------------------------------------------------------- + +test("missing token returns 401 when token is configured", async () => { + process.env.OPENCODE_LLM_PROXY_TOKEN = "secret-token" + + try { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/health") + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 401) + assert.equal(body.error.type, "invalid_request_error") + assert.ok(response.headers.get("www-authenticate")?.includes("Bearer")) + } finally { + delete process.env.OPENCODE_LLM_PROXY_TOKEN + } +}) + +test("wrong token returns 401", async () => { + process.env.OPENCODE_LLM_PROXY_TOKEN = "secret-token" + + try { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/health", { + headers: { Authorization: "Bearer wrong-token" }, + }) + + const response = await handler(request) + + assert.equal(response.status, 401) + } finally { + delete process.env.OPENCODE_LLM_PROXY_TOKEN + } +}) + +test("correct token passes through", async () => { + process.env.OPENCODE_LLM_PROXY_TOKEN = "secret-token" + + try { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/health", { + headers: { Authorization: "Bearer secret-token" }, + }) + + const response = await handler(request) + + assert.equal(response.status, 200) + } finally { + delete process.env.OPENCODE_LLM_PROXY_TOKEN + } +}) + +test("no token configured allows all requests through", async () => { + delete process.env.OPENCODE_LLM_PROXY_TOKEN + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/health") + + const response = await handler(request) + + assert.equal(response.status, 200) +}) + // --------------------------------------------------------------------------- -// Unit: toTextContent +// Integration: /v1/chat/completions error handling +// --------------------------------------------------------------------------- + +test("malformed JSON body returns 400", async () => { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{ not valid json", + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.equal(body.error.type, "invalid_request_error") +}) + +test("missing model field returns 400", async () => { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [{ role: "user", content: "hi" }] }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.ok(body.error.message.includes("model")) +}) + +test("missing messages field returns 400", async () => { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4o" }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.ok(body.error.message.includes("messages")) +}) + +test("stream: true returns 400 (not implemented)", async () => { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + messages: [{ role: "user", content: "hi" }], + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.ok(body.error.message.toLowerCase().includes("stream")) +}) + +test("unknown model returns 502", async () => { + const handler = createProxyFetchHandler(createClient()) // client returns no providers + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "nonexistent-model", + messages: [{ role: "user", content: "hi" }], + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 502) + assert.ok(body.error.message.includes("nonexistent-model")) +}) + +test("unknown route returns 404", async () => { + const handler = createProxyFetchHandler(createClient()) + const request = new Request("http://127.0.0.1:4010/unknown-path") + + const response = await handler(request) + + assert.equal(response.status, 404) +}) + // --------------------------------------------------------------------------- describe("toTextContent", () => { it("returns a string unchanged", () => { diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..210fc17 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,933 @@ +{ + "name": "opencode-llm-proxy", + "version": "1.2.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "opencode-llm-proxy", + "version": "1.2.0", + "license": "MIT", + "devDependencies": { + "@eslint/js": "^10.0.1", + "eslint": "^10.1.0" + }, + "peerDependencies": { + "opencode-ai": "*" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.3", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^3.0.3", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.5.3", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.1.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.1.1", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/js": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.3", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.6.1", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.1.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.7", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.16.0", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.14.0", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.5", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.1.0.tgz", + "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.3", + "@eslint/config-helpers": "^0.5.3", + "@eslint/core": "^1.1.1", + "@eslint/plugin-kit": "^0.6.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "9.1.2", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "11.2.0", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "dev": true, + "license": "MIT" + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.2", + "dev": true, + "license": "ISC" + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "dev": true, + "license": "ISC" + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "10.2.4", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "dev": true, + "license": "MIT" + }, + "node_modules/opencode-ai": { + "version": "1.3.3", + "hasInstallScript": true, + "license": "MIT", + "peer": true, + "bin": { + "opencode": "bin/opencode" + }, + "optionalDependencies": { + "opencode-darwin-arm64": "1.3.3", + "opencode-darwin-x64": "1.3.3", + "opencode-darwin-x64-baseline": "1.3.3", + "opencode-linux-arm64": "1.3.3", + "opencode-linux-arm64-musl": "1.3.3", + "opencode-linux-x64": "1.3.3", + "opencode-linux-x64-baseline": "1.3.3", + "opencode-linux-x64-baseline-musl": "1.3.3", + "opencode-linux-x64-musl": "1.3.3", + "opencode-windows-arm64": "1.3.3", + "opencode-windows-x64": "1.3.3", + "opencode-windows-x64-baseline": "1.3.3" + } + }, + "node_modules/opencode-darwin-arm64": { + "version": "1.3.3", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "peer": true + }, + "node_modules/opencode-darwin-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-darwin-x64/-/opencode-darwin-x64-1.3.3.tgz", + "integrity": "sha512-/AmjZ2hu7pVRKpj7t6siiiW3xo68enjRUmfAOI+grIAdX64oh+95xf/l7hsf2TLIWjRev+9kOBjUVMQTQNu2VA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "peer": true + }, + "node_modules/opencode-darwin-x64-baseline": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-darwin-x64-baseline/-/opencode-darwin-x64-baseline-1.3.3.tgz", + "integrity": "sha512-4Hp1Sr99BL3Poa+kz9ZNp0Lt9uwIoT8OYF/f10jNdMUZLBNSijVIiSH0zH3KyBKMRvNl0ZcWbOvKGTaRh9X0Kg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "peer": true + }, + "node_modules/opencode-linux-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-linux-arm64/-/opencode-linux-arm64-1.3.3.tgz", + "integrity": "sha512-i2/PR9lMPpn0RjELiAKcdLkDjtBHP+l/HVxNFB7l3E9jXT2V/WohOZOGlegIsYmm6bB10/qU0UrzPlRLLJY1kg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "peer": true + }, + "node_modules/opencode-linux-arm64-musl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-linux-arm64-musl/-/opencode-linux-arm64-musl-1.3.3.tgz", + "integrity": "sha512-N4pBzZDeTq4noc4/SwIm4roGb6OtDt9XOOE9p6OsB+4JhCuBIUcyMW71EQCIFlH197vmcJfWCo/AdCa3OS6uHA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "peer": true + }, + "node_modules/opencode-linux-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-linux-x64/-/opencode-linux-x64-1.3.3.tgz", + "integrity": "sha512-BpqYkbk8adAvnXTNFOjs5gxOsbqA/+l7J0PRIQtvslwRgVnrPMQoCXeD9okSXaVxMvyil4mWdodalY3wkY5LWg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "peer": true + }, + "node_modules/opencode-linux-x64-baseline": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-linux-x64-baseline/-/opencode-linux-x64-baseline-1.3.3.tgz", + "integrity": "sha512-9dY89V7tKNzyOsbH9pIQORCxPGImwnDvoyMZ1s1NtXDCXz/ZJWfzYOcVWBZZA7frRcwnwIueZjrz0aARDAtLdg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "peer": true + }, + "node_modules/opencode-linux-x64-baseline-musl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-linux-x64-baseline-musl/-/opencode-linux-x64-baseline-musl-1.3.3.tgz", + "integrity": "sha512-QXiIDscOCDN0z80SrO/L4Oi/f8fxs0c4zV12eUA13zw8MITLjOzdRoBIIv8jwwgjLC7rJd/PE8CIkiB5xMjuXQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "peer": true + }, + "node_modules/opencode-linux-x64-musl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-linux-x64-musl/-/opencode-linux-x64-musl-1.3.3.tgz", + "integrity": "sha512-mJlzR+VOv+zqxLbpd4JhUF9ElbN/9ebQ395onTUDZU3vGtTvqz5Z3cZm8R7Xd+GNs5f/AkPUNyYqlHrmT85RKw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "peer": true + }, + "node_modules/opencode-windows-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-windows-arm64/-/opencode-windows-arm64-1.3.3.tgz", + "integrity": "sha512-1GeiiZocPzE0mBp6cgON/180DN1v+jT0YH4mKEY1nof5V0CWS5WazvciPdGDTf0mfnvz+FfrDpxFxpA8HVU+SA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "peer": true + }, + "node_modules/opencode-windows-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-windows-x64/-/opencode-windows-x64-1.3.3.tgz", + "integrity": "sha512-pE7VJNy3s3nMgdhbbZIGW9f/kHxgdV3sqAxM5kJd8WVOetQQ7DxUH52+rwYs0DqyoX9lZqIY2SnWCRFxio5Qtw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "peer": true + }, + "node_modules/opencode-windows-x64-baseline": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/opencode-windows-x64-baseline/-/opencode-windows-x64-baseline-1.3.3.tgz", + "integrity": "sha512-+S6ADlSdB3Cf+JfkVeJ1087lM6BeCslROfNUt3I2Y6hktqwOKRnlhI/dz/SySaGeQD0gysgYcQ7PzZPQpPNa/w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "peer": true + }, + "node_modules/optionator": { + "version": "0.9.4", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/package.json b/package.json index f67e23b..a20f16c 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "type": "module", "scripts": { "test": "node --test --experimental-test-coverage", + "lint": "eslint .", "start": "node index.js" }, "keywords": [ @@ -23,5 +24,9 @@ }, "peerDependencies": { "opencode-ai": "*" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "eslint": "^10.1.0" } } From cdaba5ddd4702e63ff83e885c539aaf4d62ebb63 Mon Sep 17 00:00:00 2001 From: KochC Date: Fri, 27 Mar 2026 16:08:57 +0100 Subject: [PATCH 02/11] feat: implement SSE streaming and support all opencode providers - Implement streaming for POST /v1/chat/completions (issue #11): subscribe to opencode event stream, pipe message.part.updated deltas as SSE chat.completion.chunk events, finish on session.idle - Implement streaming for POST /v1/responses (issue #11): emit response.created / output_text.delta / response.completed events - Fix provider-agnostic system prompt hint (issue #12): remove 'OpenAI-compatible' wording so non-OpenAI models are not confused - Add TextEncoder and ReadableStream to ESLint globals - Add streaming integration tests (happy path, unknown model, session.error) --- eslint.config.js | 2 + index.js | 395 ++++++++++++++++++++++++++++++++++++++++++++--- index.test.js | 125 ++++++++++++++- 3 files changed, 495 insertions(+), 27 deletions(-) diff --git a/eslint.config.js b/eslint.config.js index 45f5a7e..b887cf9 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -17,6 +17,8 @@ export default [ Request: "readonly", Response: "readonly", URL: "readonly", + TextEncoder: "readonly", + ReadableStream: "readonly", }, }, rules: { diff --git a/index.js b/index.js index c00a412..3c905fb 100644 --- a/index.js +++ b/index.js @@ -178,7 +178,7 @@ export function buildSystemPrompt(messages, request) { .map((message) => message.content) const hints = [ - "You are answering through an OpenAI-compatible proxy backed by OpenCode.", + "You are answering through a proxy backed by OpenCode.", "Return only the assistant's reply content.", ] @@ -268,6 +268,69 @@ async function executePrompt(client, request, model, messages, system) { } } +async function executePromptStreaming(client, model, messages, system, onChunk) { + const tools = await getDisabledTools(client) + const session = await client.session.create({ + body: { title: `Proxy: ${model.id}` }, + }) + const sessionID = session.data.id + const prompt = buildPrompt(messages) + + // Subscribe to the event stream before sending the prompt so we don't miss events. + const { stream } = await client.event.subscribe() + + await client.session.promptAsync({ + path: { id: sessionID }, + body: { + model: { providerID: model.providerID, modelID: model.modelID }, + system, + tools, + parts: [{ type: "text", text: prompt }], + }, + }) + + let errorMessage = null + + for await (const event of stream) { + if (event.type === "message.part.updated") { + const part = event.properties?.part + const delta = event.properties?.delta + if ( + part?.sessionID === sessionID && + part?.type === "text" && + typeof delta === "string" && + delta.length > 0 + ) { + onChunk(delta) + } + } else if (event.type === "session.error") { + if (!event.properties?.sessionID || event.properties.sessionID === sessionID) { + errorMessage = event.properties?.error?.message ?? "Model call failed." + } + } else if (event.type === "session.idle") { + if (event.properties?.sessionID === sessionID) { + break + } + } + } + + if (errorMessage) { + throw new Error(errorMessage) + } + + // Fetch final message to get token usage. + const messages_ = await client.session.messages({ path: { id: sessionID } }) + const assistantMsg = (messages_.data ?? []) + .filter((m) => m.role === "assistant") + .at(-1) + + return { + sessionID, + tokens: assistantMsg?.tokens ?? { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + finish: assistantMsg?.finish, + } +} + function createChatCompletionResponse(result, model) { const now = Math.floor(Date.now() / 1000) return { @@ -424,6 +487,33 @@ export async function resolveModel(client, requestedModel, providerOverride) { throw new Error(`Unknown model '${requestedModel}'. Call GET /v1/models to inspect available IDs.`) } +function sseResponse(corsHeadersObj, generator) { + const encoder = new TextEncoder() + const body = new ReadableStream({ + async start(controller) { + try { + for await (const chunk of generator) { + controller.enqueue(encoder.encode(chunk)) + } + } catch { + // Stream errors are surfaced via SSE data before this point. + } finally { + controller.close() + } + }, + }) + + return new Response(body, { + status: 200, + headers: { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache", + connection: "keep-alive", + ...corsHeadersObj, + }, + }) +} + function createModelResponse(models) { return { object: "list", @@ -473,10 +563,6 @@ export function createProxyFetchHandler(client) { return badRequest("Request body must be valid JSON.", 400, request) } - if (body.stream) { - return badRequest("Streaming is not implemented yet.", 400, request) - } - if (!body.model) { return badRequest("The 'model' field is required.", 400, request) } @@ -490,10 +576,111 @@ export function createProxyFetchHandler(client) { return badRequest("No text content was found in the supplied messages.", 400, request) } + let model try { const providerOverride = request.headers.get("x-opencode-provider") - const model = await resolveModel(client, body.model, providerOverride) - const system = buildSystemPrompt(messages, body) + model = await resolveModel(client, body.model, providerOverride) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + await safeLog(client, "error", "Proxy completion failed", { + error: message, + requestedModel: body.model, + }) + return badRequest(message, 502, request) + } + + const system = buildSystemPrompt(messages, body) + + if (body.stream) { + const completionID = `chatcmpl_${crypto.randomUUID().replace(/-/g, "")}` + const now = Math.floor(Date.now() / 1000) + + const chunks = [] + let resolve = null + let done = false + + function enqueue(value) { + chunks.push(value) + if (resolve) { + const r = resolve + resolve = null + r() + } + } + + async function* generateSse() { + const runPromise = executePromptStreaming( + client, + model, + messages, + system, + (delta) => { + const chunk = JSON.stringify({ + id: completionID, + object: "chat.completion.chunk", + created: now, + model: model.id, + choices: [{ index: 0, delta: { role: "assistant", content: delta }, finish_reason: null }], + }) + enqueue(`data: ${chunk}\n\n`) + }, + ) + .then((streamResult) => { + const finalChunk = JSON.stringify({ + id: completionID, + object: "chat.completion.chunk", + created: now, + model: model.id, + choices: [{ index: 0, delta: {}, finish_reason: mapFinishReason(streamResult.finish) }], + usage: { + prompt_tokens: streamResult.tokens.input, + completion_tokens: streamResult.tokens.output, + total_tokens: streamResult.tokens.input + streamResult.tokens.output, + }, + }) + enqueue(`data: ${finalChunk}\n\ndata: [DONE]\n\n`) + }) + .catch(async (err) => { + const streamError = err instanceof Error ? err.message : String(err) + await safeLog(client, "error", "Proxy streaming completion failed", { + error: streamError, + requestedModel: body.model, + }) + const errChunk = JSON.stringify({ + error: { message: streamError, type: "server_error" }, + }) + enqueue(`data: ${errChunk}\n\ndata: [DONE]\n\n`) + }) + .finally(() => { + done = true + if (resolve) { + const r = resolve + resolve = null + r() + } + }) + + while (true) { + while (chunks.length > 0) { + yield chunks.shift() + } + if (done) break + await new Promise((r) => { + resolve = r + }) + } + // Drain any remaining chunks + while (chunks.length > 0) { + yield chunks.shift() + } + + await runPromise + } + + return sseResponse(corsHeaders(request), generateSse()) + } + + try { const result = await executePrompt(client, body, model, messages, system) return json(createChatCompletionResponse(result, model), 200, {}, request) } catch (error) { @@ -514,10 +701,6 @@ export function createProxyFetchHandler(client) { return badRequest("Request body must be valid JSON.", 400, request) } - if (body.stream) { - return badRequest("Streaming is not implemented yet.", 400, request) - } - if (!body.model) { return badRequest("The 'model' field is required.", 400, request) } @@ -527,19 +710,187 @@ export function createProxyFetchHandler(client) { return badRequest("The 'input' field must contain at least one text message.", 400, request) } + const instructionMessages = + typeof body.instructions === "string" && body.instructions.trim() + ? [{ role: "system", content: body.instructions.trim() }, ...messages] + : messages + + const system = buildSystemPrompt(instructionMessages, { + temperature: body.temperature, + max_tokens: body.max_output_tokens, + max_completion_tokens: body.max_output_tokens, + }) + + let model try { const providerOverride = request.headers.get("x-opencode-provider") - const model = await resolveModel(client, body.model, providerOverride) - const system = buildSystemPrompt( - typeof body.instructions === "string" && body.instructions.trim() - ? [{ role: "system", content: body.instructions.trim() }, ...messages] - : messages, - { - temperature: body.temperature, - max_tokens: body.max_output_tokens, - max_completion_tokens: body.max_output_tokens, - }, - ) + model = await resolveModel(client, body.model, providerOverride) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + await safeLog(client, "error", "Proxy responses call failed", { + error: message, + requestedModel: body.model, + }) + return badRequest(message, 502, request) + } + + if (body.stream) { + const responseID = `resp_${crypto.randomUUID().replace(/-/g, "")}` + const itemID = `msg_${crypto.randomUUID().replace(/-/g, "")}` + const now = Math.floor(Date.now() / 1000) + + const chunks = [] + let resolve = null + let done = false + + function enqueue(value) { + chunks.push(value) + if (resolve) { + const r = resolve + resolve = null + r() + } + } + + function sseEvent(eventType, data) { + return `event: ${eventType}\ndata: ${JSON.stringify(data)}\n\n` + } + + async function* generateSse() { + enqueue( + sseEvent("response.created", { + type: "response.created", + response: { + id: responseID, + object: "response", + created_at: now, + status: "in_progress", + model: model.id, + output: [], + }, + }), + ) + enqueue( + sseEvent("response.output_item.added", { + type: "response.output_item.added", + output_index: 0, + item: { id: itemID, type: "message", status: "in_progress", role: "assistant", content: [] }, + }), + ) + + let partIndex = 0 + const runPromise = executePromptStreaming( + client, + model, + messages, + system, + (delta) => { + if (partIndex === 0) { + enqueue( + sseEvent("response.content_part.added", { + type: "response.content_part.added", + item_id: itemID, + output_index: 0, + content_index: 0, + part: { type: "output_text", text: "", annotations: [] }, + }), + ) + partIndex++ + } + enqueue( + sseEvent("response.output_text.delta", { + type: "response.output_text.delta", + item_id: itemID, + output_index: 0, + content_index: 0, + delta, + }), + ) + }, + ) + .then((streamResult) => { + enqueue( + sseEvent("response.output_text.done", { + type: "response.output_text.done", + item_id: itemID, + output_index: 0, + content_index: 0, + text: "", + }), + ) + enqueue( + sseEvent("response.output_item.done", { + type: "response.output_item.done", + output_index: 0, + item: { id: itemID, type: "message", status: "completed", role: "assistant" }, + }), + ) + enqueue( + sseEvent("response.completed", { + type: "response.completed", + response: { + id: responseID, + object: "response", + created_at: now, + status: "completed", + model: model.id, + usage: { + input_tokens: streamResult.tokens.input, + output_tokens: streamResult.tokens.output, + total_tokens: streamResult.tokens.input + streamResult.tokens.output, + }, + }, + }), + ) + }) + .catch(async (err) => { + const errMsg = err instanceof Error ? err.message : String(err) + await safeLog(client, "error", "Proxy streaming responses call failed", { + error: errMsg, + requestedModel: body.model, + }) + enqueue( + sseEvent("response.failed", { + type: "response.failed", + response: { + id: responseID, + object: "response", + created_at: now, + status: "failed", + error: { message: errMsg, code: "server_error" }, + }, + }), + ) + }) + .finally(() => { + done = true + if (resolve) { + const r = resolve + resolve = null + r() + } + }) + + while (true) { + while (chunks.length > 0) { + yield chunks.shift() + } + if (done) break + await new Promise((r) => { + resolve = r + }) + } + while (chunks.length > 0) { + yield chunks.shift() + } + + await runPromise + } + + return sseResponse(corsHeaders(request), generateSse()) + } + + try { const result = await executePrompt(client, body, model, messages, system) return json(createResponsesApiResponse(result, model), 200, {}, request) } catch (error) { diff --git a/index.test.js b/index.test.js index e4241f7..264f53c 100644 --- a/index.test.js +++ b/index.test.js @@ -32,6 +32,47 @@ function createClient() { } } +function createStreamingClient(chunks) { + async function* makeStream() { + for (const chunk of chunks) { + yield chunk + } + } + + return { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [ + { + id: "openai", + models: { "gpt-4o": { id: "gpt-4o", name: "GPT-4o" } }, + }, + ], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-123" } }), + promptAsync: async () => {}, + messages: async () => ({ + data: [ + { + role: "assistant", + tokens: { input: 10, output: 5, reasoning: 0, cache: { read: 0, write: 0 } }, + finish: "end_turn", + }, + ], + }), + }, + event: { + subscribe: async () => ({ stream: makeStream() }), + }, + } +} + test("OPTIONS preflight returns CORS headers", async () => { const handler = createProxyFetchHandler(createClient()) const request = new Request("http://127.0.0.1:4010/v1/models", { @@ -260,8 +301,26 @@ test("missing messages field returns 400", async () => { assert.ok(body.error.message.includes("messages")) }) -test("stream: true returns 400 (not implemented)", async () => { - const handler = createProxyFetchHandler(createClient()) +test("stream: true returns SSE response", async () => { + const events = [ + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: "Hello", + }, + }, + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: " world", + }, + }, + { type: "session.idle", properties: { sessionID: "sess-123" } }, + ] + + const handler = createProxyFetchHandler(createStreamingClient(events)) const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json" }, @@ -272,11 +331,67 @@ test("stream: true returns 400 (not implemented)", async () => { }), }) + const response = await handler(request) + + assert.equal(response.status, 200) + assert.ok(response.headers.get("content-type")?.includes("text/event-stream")) + + const text = await response.text() + assert.ok(text.includes("chat.completion.chunk")) + assert.ok(text.includes("Hello")) + assert.ok(text.includes(" world")) + assert.ok(text.includes("[DONE]")) +}) + +test("stream: true with unknown model returns 502", async () => { + const handler = createProxyFetchHandler(createClient()) // no providers + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "nonexistent-model", + stream: true, + messages: [{ role: "user", content: "hi" }], + }), + }) + const response = await handler(request) const body = await response.json() - assert.equal(response.status, 400) - assert.ok(body.error.message.toLowerCase().includes("stream")) + assert.equal(response.status, 502) + assert.ok(body.error.message.includes("nonexistent-model")) +}) + +test("stream: true propagates session.error into the SSE stream", async () => { + const events = [ + { + type: "session.error", + properties: { + sessionID: "sess-123", + error: { message: "Model overloaded" }, + }, + }, + { type: "session.idle", properties: { sessionID: "sess-123" } }, + ] + + const handler = createProxyFetchHandler(createStreamingClient(events)) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + messages: [{ role: "user", content: "hi" }], + }), + }) + + const response = await handler(request) + assert.equal(response.status, 200) + assert.ok(response.headers.get("content-type")?.includes("text/event-stream")) + + const text = await response.text() + assert.ok(text.includes("server_error") || text.includes("Model overloaded")) + assert.ok(text.includes("[DONE]")) }) test("unknown model returns 502", async () => { @@ -446,7 +561,7 @@ describe("buildSystemPrompt", () => { it("always includes the proxy hint lines", () => { const result = buildSystemPrompt([], {}) - assert.ok(result.includes("OpenAI-compatible proxy")) + assert.ok(result.includes("proxy backed by OpenCode")) assert.ok(result.includes("Return only the assistant")) }) From 41b05727570fb6848bb3507901e774c9ba7d2a50 Mon Sep 17 00:00:00 2001 From: KochC Date: Fri, 27 Mar 2026 16:42:56 +0100 Subject: [PATCH 03/11] feat: refactor SSE queue, expand test coverage, fix package metadata - Extract createSseQueue() helper, eliminating duplicated SSE queue pattern in /v1/chat/completions and /v1/responses streaming branches (closes #34) - Add tests for GET /v1/models happy path, empty providers, and error path (closes #33) - Add tests for POST /v1/responses: happy path, validation, streaming, session.error (closes #32) - Fix package.json description to be provider-agnostic (closes #35) - Add engines field declaring bun >=1.0.0 requirement (closes #35) - Line coverage: 55% -> 89%, function coverage: 83% -> 94% --- index.js | 131 ++++++++---------- index.test.js | 369 ++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 5 +- 3 files changed, 432 insertions(+), 73 deletions(-) diff --git a/index.js b/index.js index 3c905fb..c9c1918 100644 --- a/index.js +++ b/index.js @@ -487,6 +487,48 @@ export async function resolveModel(client, requestedModel, providerOverride) { throw new Error(`Unknown model '${requestedModel}'. Call GET /v1/models to inspect available IDs.`) } +export function createSseQueue() { + const chunks = [] + let resolve = null + let done = false + + function enqueue(value) { + chunks.push(value) + if (resolve) { + const r = resolve + resolve = null + r() + } + } + + function finish() { + done = true + if (resolve) { + const r = resolve + resolve = null + r() + } + } + + async function* generateChunks() { + while (true) { + while (chunks.length > 0) { + yield chunks.shift() + } + if (done) break + await new Promise((r) => { + resolve = r + }) + } + // Drain any remaining chunks + while (chunks.length > 0) { + yield chunks.shift() + } + } + + return { enqueue, finish, generateChunks } +} + function sseResponse(corsHeadersObj, generator) { const encoder = new TextEncoder() const body = new ReadableStream({ @@ -595,18 +637,7 @@ export function createProxyFetchHandler(client) { const completionID = `chatcmpl_${crypto.randomUUID().replace(/-/g, "")}` const now = Math.floor(Date.now() / 1000) - const chunks = [] - let resolve = null - let done = false - - function enqueue(value) { - chunks.push(value) - if (resolve) { - const r = resolve - resolve = null - r() - } - } + const queue = createSseQueue() async function* generateSse() { const runPromise = executePromptStreaming( @@ -622,7 +653,7 @@ export function createProxyFetchHandler(client) { model: model.id, choices: [{ index: 0, delta: { role: "assistant", content: delta }, finish_reason: null }], }) - enqueue(`data: ${chunk}\n\n`) + queue.enqueue(`data: ${chunk}\n\n`) }, ) .then((streamResult) => { @@ -638,7 +669,7 @@ export function createProxyFetchHandler(client) { total_tokens: streamResult.tokens.input + streamResult.tokens.output, }, }) - enqueue(`data: ${finalChunk}\n\ndata: [DONE]\n\n`) + queue.enqueue(`data: ${finalChunk}\n\ndata: [DONE]\n\n`) }) .catch(async (err) => { const streamError = err instanceof Error ? err.message : String(err) @@ -649,30 +680,13 @@ export function createProxyFetchHandler(client) { const errChunk = JSON.stringify({ error: { message: streamError, type: "server_error" }, }) - enqueue(`data: ${errChunk}\n\ndata: [DONE]\n\n`) + queue.enqueue(`data: ${errChunk}\n\ndata: [DONE]\n\n`) }) .finally(() => { - done = true - if (resolve) { - const r = resolve - resolve = null - r() - } + queue.finish() }) - while (true) { - while (chunks.length > 0) { - yield chunks.shift() - } - if (done) break - await new Promise((r) => { - resolve = r - }) - } - // Drain any remaining chunks - while (chunks.length > 0) { - yield chunks.shift() - } + yield* queue.generateChunks() await runPromise } @@ -739,25 +753,14 @@ export function createProxyFetchHandler(client) { const itemID = `msg_${crypto.randomUUID().replace(/-/g, "")}` const now = Math.floor(Date.now() / 1000) - const chunks = [] - let resolve = null - let done = false - - function enqueue(value) { - chunks.push(value) - if (resolve) { - const r = resolve - resolve = null - r() - } - } + const queue = createSseQueue() function sseEvent(eventType, data) { return `event: ${eventType}\ndata: ${JSON.stringify(data)}\n\n` } async function* generateSse() { - enqueue( + queue.enqueue( sseEvent("response.created", { type: "response.created", response: { @@ -770,7 +773,7 @@ export function createProxyFetchHandler(client) { }, }), ) - enqueue( + queue.enqueue( sseEvent("response.output_item.added", { type: "response.output_item.added", output_index: 0, @@ -786,7 +789,7 @@ export function createProxyFetchHandler(client) { system, (delta) => { if (partIndex === 0) { - enqueue( + queue.enqueue( sseEvent("response.content_part.added", { type: "response.content_part.added", item_id: itemID, @@ -797,7 +800,7 @@ export function createProxyFetchHandler(client) { ) partIndex++ } - enqueue( + queue.enqueue( sseEvent("response.output_text.delta", { type: "response.output_text.delta", item_id: itemID, @@ -809,7 +812,7 @@ export function createProxyFetchHandler(client) { }, ) .then((streamResult) => { - enqueue( + queue.enqueue( sseEvent("response.output_text.done", { type: "response.output_text.done", item_id: itemID, @@ -818,14 +821,14 @@ export function createProxyFetchHandler(client) { text: "", }), ) - enqueue( + queue.enqueue( sseEvent("response.output_item.done", { type: "response.output_item.done", output_index: 0, item: { id: itemID, type: "message", status: "completed", role: "assistant" }, }), ) - enqueue( + queue.enqueue( sseEvent("response.completed", { type: "response.completed", response: { @@ -849,7 +852,7 @@ export function createProxyFetchHandler(client) { error: errMsg, requestedModel: body.model, }) - enqueue( + queue.enqueue( sseEvent("response.failed", { type: "response.failed", response: { @@ -863,26 +866,10 @@ export function createProxyFetchHandler(client) { ) }) .finally(() => { - done = true - if (resolve) { - const r = resolve - resolve = null - r() - } + queue.finish() }) - while (true) { - while (chunks.length > 0) { - yield chunks.shift() - } - if (done) break - await new Promise((r) => { - resolve = r - }) - } - while (chunks.length > 0) { - yield chunks.shift() - } + yield* queue.generateChunks() await runPromise } diff --git a/index.test.js b/index.test.js index 264f53c..f26b033 100644 --- a/index.test.js +++ b/index.test.js @@ -3,6 +3,7 @@ import assert from "node:assert/strict" import { createProxyFetchHandler, + createSseQueue, toTextContent, normalizeMessages, normalizeResponseInput, @@ -762,3 +763,371 @@ describe("resolveModel", () => { assert.equal(model.modelID, "gpt-4o-mini") }) }) + +// --------------------------------------------------------------------------- +// Unit: createSseQueue +// --------------------------------------------------------------------------- +describe("createSseQueue", () => { + it("enqueue followed by generateChunks yields the value", async () => { + const queue = createSseQueue() + queue.enqueue("hello") + queue.finish() + const results = [] + for await (const chunk of queue.generateChunks()) { + results.push(chunk) + } + assert.deepEqual(results, ["hello"]) + }) + + it("multiple enqueues before finish yields all values in order", async () => { + const queue = createSseQueue() + queue.enqueue("a") + queue.enqueue("b") + queue.enqueue("c") + queue.finish() + const results = [] + for await (const chunk of queue.generateChunks()) { + results.push(chunk) + } + assert.deepEqual(results, ["a", "b", "c"]) + }) + + it("finish with no enqueues yields nothing", async () => { + const queue = createSseQueue() + queue.finish() + const results = [] + for await (const chunk of queue.generateChunks()) { + results.push(chunk) + } + assert.deepEqual(results, []) + }) + + it("enqueue after generateChunks starts still yields the value", async () => { + const queue = createSseQueue() + // Start consuming before anything is enqueued + const generatorPromise = (async () => { + const results = [] + for await (const chunk of queue.generateChunks()) { + results.push(chunk) + } + return results + })() + // Enqueue asynchronously + await Promise.resolve() + queue.enqueue("late") + queue.finish() + const results = await generatorPromise + assert.deepEqual(results, ["late"]) + }) +}) + +// --------------------------------------------------------------------------- +// Integration: GET /v1/models +// --------------------------------------------------------------------------- + +function createModelsClient(providers = []) { + return { + app: { log: async () => {} }, + config: { + providers: async () => ({ data: { providers } }), + }, + } +} + +test("GET /v1/models returns model list", async () => { + const client = createModelsClient([ + { + id: "openai", + models: { + "gpt-4o": { id: "gpt-4o", name: "GPT-4o" }, + "gpt-4o-mini": { id: "gpt-4o-mini", name: "GPT-4o Mini" }, + }, + }, + { + id: "anthropic", + models: { + "claude-3-5-sonnet": { id: "claude-3-5-sonnet", name: "Claude 3.5 Sonnet" }, + }, + }, + ]) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/models") + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.equal(body.object, "list") + assert.ok(Array.isArray(body.data)) + assert.equal(body.data.length, 3) + + const ids = body.data.map((m) => m.id) + assert.ok(ids.includes("openai/gpt-4o")) + assert.ok(ids.includes("openai/gpt-4o-mini")) + assert.ok(ids.includes("anthropic/claude-3-5-sonnet")) + + const first = body.data[0] + assert.equal(first.object, "model") + assert.ok("owned_by" in first) + assert.ok("created" in first) +}) + +test("GET /v1/models returns empty list when no providers configured", async () => { + const handler = createProxyFetchHandler(createModelsClient([])) + const request = new Request("http://127.0.0.1:4010/v1/models") + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.deepEqual(body, { object: "list", data: [] }) +}) + +test("GET /v1/models returns 500 when providers call throws", async () => { + const client = { + app: { log: async () => {} }, + config: { + providers: async () => { + throw new Error("upstream failure") + }, + }, + } + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/models") + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 500) + assert.equal(body.error.type, "server_error") +}) + +// --------------------------------------------------------------------------- +// Integration: POST /v1/responses +// --------------------------------------------------------------------------- + +function createResponsesClient(responseContent = "The answer is 42.") { + return { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [ + { + id: "anthropic", + models: { "claude-3-5-sonnet": { id: "claude-3-5-sonnet", name: "Claude 3.5 Sonnet" } }, + }, + ], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-resp-1" } }), + prompt: async () => ({ + data: { + parts: [{ type: "text", text: responseContent }], + info: { tokens: { input: 20, output: 8, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "end_turn" }, + }, + }), + }, + } +} + +test("POST /v1/responses returns a well-formed response object", async () => { + const handler = createProxyFetchHandler(createResponsesClient("Hello from Claude.")) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "anthropic/claude-3-5-sonnet", + input: "Say hello.", + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.equal(body.object, "response") + assert.equal(body.status, "completed") + assert.ok(body.id.startsWith("resp_")) + assert.equal(body.output_text, "Hello from Claude.") + assert.ok(Array.isArray(body.output)) + assert.equal(body.output[0].role, "assistant") + assert.equal(body.usage.input_tokens, 20) + assert.equal(body.usage.output_tokens, 8) + assert.equal(body.usage.total_tokens, 28) +}) + +test("POST /v1/responses missing model returns 400", async () => { + const handler = createProxyFetchHandler(createResponsesClient()) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ input: "hi" }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.ok(body.error.message.includes("model")) +}) + +test("POST /v1/responses empty input returns 400", async () => { + const handler = createProxyFetchHandler(createResponsesClient()) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-3-5-sonnet", input: " " }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.ok(body.error.message.includes("input")) +}) + +test("POST /v1/responses malformed JSON returns 400", async () => { + const handler = createProxyFetchHandler(createResponsesClient()) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{ bad json", + }) + + const response = await handler(request) + + assert.equal(response.status, 400) +}) + +test("POST /v1/responses unknown model returns 502", async () => { + const handler = createProxyFetchHandler(createModelsClient([])) // no providers + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "nonexistent", input: "hi" }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 502) + assert.ok(body.error.message.includes("nonexistent")) +}) + +test("POST /v1/responses instructions field is incorporated", async () => { + let capturedSystem = null + const client = { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [{ id: "anthropic", models: { "claude-3-5-sonnet": { id: "claude-3-5-sonnet" } } }], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-instr" } }), + prompt: async ({ body }) => { + capturedSystem = body.system + return { + data: { + parts: [{ type: "text", text: "ok" }], + info: { tokens: { input: 1, output: 1, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "end_turn" }, + }, + } + }, + }, + } + + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "anthropic/claude-3-5-sonnet", + input: "What is 2+2?", + instructions: "You are a math tutor.", + }), + }) + + await handler(request) + assert.ok(capturedSystem?.includes("You are a math tutor.")) +}) + +test("POST /v1/responses stream: true returns SSE lifecycle events", async () => { + const events = [ + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: "The answer", + }, + }, + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: " is 42.", + }, + }, + { type: "session.idle", properties: { sessionID: "sess-123" } }, + ] + + const handler = createProxyFetchHandler(createStreamingClient(events)) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + input: "What is 6 times 7?", + }), + }) + + const response = await handler(request) + + assert.equal(response.status, 200) + assert.ok(response.headers.get("content-type")?.includes("text/event-stream")) + + const text = await response.text() + assert.ok(text.includes("response.created")) + assert.ok(text.includes("response.output_text.delta")) + assert.ok(text.includes("The answer")) + assert.ok(text.includes(" is 42.")) + assert.ok(text.includes("response.completed")) +}) + +test("POST /v1/responses stream: true with session.error emits response.failed", async () => { + const events = [ + { + type: "session.error", + properties: { + sessionID: "sess-123", + error: { message: "Rate limit exceeded" }, + }, + }, + { type: "session.idle", properties: { sessionID: "sess-123" } }, + ] + + const handler = createProxyFetchHandler(createStreamingClient(events)) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + input: "hi", + }), + }) + + const response = await handler(request) + assert.equal(response.status, 200) + + const text = await response.text() + assert.ok(text.includes("response.failed") || text.includes("Rate limit exceeded")) +}) diff --git a/package.json b/package.json index 2edf778..5e0d820 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,12 @@ { "name": "opencode-llm-proxy", "version": "1.3.0", - "description": "OpenCode plugin that exposes an OpenAI-compatible HTTP proxy backed by your OpenCode providers", + "description": "OpenCode plugin that exposes an OpenAI-compatible HTTP proxy backed by any LLM provider configured in OpenCode", "main": "index.js", "type": "module", + "engines": { + "bun": ">=1.0.0" + }, "scripts": { "test": "node --test --experimental-test-coverage", "lint": "eslint .", From e42eee3abfa2e54c00cf8636f03fd3c3723780db Mon Sep 17 00:00:00 2001 From: KochC Date: Fri, 27 Mar 2026 17:07:37 +0100 Subject: [PATCH 04/11] feat: add Anthropic Messages API and Google Gemini API endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST /v1/messages — Anthropic Messages API with streaming (SSE) - POST /v1beta/models/:model:generateContent — Gemini non-streaming - POST /v1beta/models/:model:streamGenerateContent — Gemini NDJSON streaming - New helpers: normalizeAnthropicMessages, normalizeGeminiContents, extractGeminiSystemInstruction, mapFinishReasonToAnthropic/Gemini - 35 new tests (77 -> 112 total, all passing) - Update README to document all supported API formats Closes #38, #39 --- README.md | 80 +++++++- index.js | 368 +++++++++++++++++++++++++++++++++ index.test.js | 555 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 996 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 90a676b..f85a06f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # opencode-llm-proxy -An [OpenCode](https://opencode.ai) plugin that starts a local OpenAI-compatible HTTP server backed by your OpenCode providers. +An [OpenCode](https://opencode.ai) plugin that starts a local HTTP server backed by your OpenCode providers, with support for multiple LLM API formats: -Any tool or application that speaks the OpenAI Chat Completions or Responses API can use it — including LangChain, custom scripts, local frontends, etc. +- **OpenAI** Chat Completions (`POST /v1/chat/completions`) and Responses (`POST /v1/responses`) +- **Anthropic** Messages API (`POST /v1/messages`) +- **Google Gemini** API (`POST /v1beta/models/:model:generateContent`) + +Any tool or SDK that targets one of these APIs can point at the proxy without code changes. ## Quickstart @@ -92,7 +96,7 @@ curl http://127.0.0.1:4010/v1/models Returns all models from all providers configured in your OpenCode setup (e.g. `github-copilot/claude-sonnet-4.6`, `ollama/qwen3.5:9b`, etc.). -### Chat completions +### OpenAI Chat Completions ```bash curl http://127.0.0.1:4010/v1/chat/completions \ @@ -105,7 +109,7 @@ curl http://127.0.0.1:4010/v1/chat/completions \ }' ``` -Use the fully-qualified `provider/model` ID from `GET /v1/models`. +Use the fully-qualified `provider/model` ID from `GET /v1/models`. Supports `"stream": true` for SSE streaming. ### OpenAI Responses API @@ -118,6 +122,69 @@ curl http://127.0.0.1:4010/v1/responses \ }' ``` +Supports `"stream": true` for SSE streaming. + +### Anthropic Messages API + +Point the Anthropic SDK (or any client) at this proxy: + +```bash +curl http://127.0.0.1:4010/v1/messages \ + -H "Content-Type: application/json" \ + -d '{ + "model": "anthropic/claude-3-5-sonnet", + "max_tokens": 1024, + "system": "You are a helpful assistant.", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +Supports `"stream": true` for SSE streaming with standard Anthropic streaming events (`message_start`, `content_block_delta`, `message_stop`, etc.). + +To point the official Anthropic SDK at this proxy: + +```js +import Anthropic from "@anthropic-ai/sdk" + +const client = new Anthropic({ + baseURL: "http://127.0.0.1:4010", + apiKey: "unused", // or your OPENCODE_LLM_PROXY_TOKEN +}) +``` + +### Google Gemini API + +```bash +# Non-streaming +curl http://127.0.0.1:4010/v1beta/models/google/gemini-2.0-flash:generateContent \ + -H "Content-Type: application/json" \ + -d '{ + "contents": [{"role": "user", "parts": [{"text": "Hello!"}]}] + }' + +# Streaming (newline-delimited JSON) +curl http://127.0.0.1:4010/v1beta/models/google/gemini-2.0-flash:streamGenerateContent \ + -H "Content-Type: application/json" \ + -d '{ + "contents": [{"role": "user", "parts": [{"text": "Hello!"}]}] + }' +``` + +The model name in the URL path is resolved the same way as other endpoints (use `provider/model` or a bare model ID if unambiguous). + +To point the Google Generative AI SDK at this proxy, set the `baseUrl` option to `http://127.0.0.1:4010`. + +## Selecting a provider + +All endpoints accept an optional `x-opencode-provider` header to force a specific provider when the model ID is ambiguous: + +```bash +curl http://127.0.0.1:4010/v1/chat/completions \ + -H "x-opencode-provider: anthropic" \ + -H "Content-Type: application/json" \ + -d '{"model": "claude-3-5-sonnet", "messages": [...]}' +``` + ## Configuration All configuration is done through environment variables. No configuration file is needed. @@ -149,15 +216,14 @@ curl http://:4010/v1/models \ ## How it works -The plugin hooks into OpenCode at startup and spawns a Bun HTTP server. Incoming OpenAI-format requests are translated into OpenCode SDK calls (`client.session.create` + `client.session.prompt`), routed through whichever provider/model is requested, and the response is returned in OpenAI format. +The plugin hooks into OpenCode at startup and spawns a Bun HTTP server. Incoming requests (in OpenAI, Anthropic, or Gemini format) are translated into OpenCode SDK calls (`client.session.create` + `client.session.prompt`), routed through whichever provider/model is requested, and the response is returned in the matching API format. Each request creates a temporary OpenCode session, so prompts and responses appear in the OpenCode session list. ## Limitations -- Streaming (`"stream": true`) is not yet implemented — requests will return a 400 error. - Tool/function calling is not forwarded; all built-in OpenCode tools are disabled for proxy sessions. -- The proxy only handles `POST /v1/chat/completions` and `POST /v1/responses`. Other OpenAI endpoints are not implemented. +- Only text content is handled; image and file inputs are ignored. ## License diff --git a/index.js b/index.js index c9c1918..227f01e 100644 --- a/index.js +++ b/index.js @@ -569,6 +569,130 @@ function createModelResponse(models) { } } +// --------------------------------------------------------------------------- +// Anthropic Messages API helpers +// --------------------------------------------------------------------------- + +export function normalizeAnthropicMessages(messages) { + return messages + .map((message) => { + let content = "" + if (typeof message.content === "string") { + content = message.content.trim() + } else if (Array.isArray(message.content)) { + content = message.content + .filter((block) => block && block.type === "text" && typeof block.text === "string") + .map((block) => block.text.trim()) + .filter(Boolean) + .join("\n\n") + } + return { role: message.role, content } + }) + .filter((message) => message.content.length > 0) +} + +export function mapFinishReasonToAnthropic(finish) { + if (!finish) return "end_turn" + if (finish.includes("length")) return "max_tokens" + if (finish.includes("tool")) return "tool_use" + return "end_turn" +} + +function createAnthropicResponse(result, model) { + const tokensIn = result.completion.data.info?.tokens?.input ?? 0 + const tokensOut = result.completion.data.info?.tokens?.output ?? 0 + return { + id: `msg_${crypto.randomUUID().replace(/-/g, "")}`, + type: "message", + role: "assistant", + content: [{ type: "text", text: result.content }], + model: model.id, + stop_reason: mapFinishReasonToAnthropic(result.completion.data.info?.finish), + stop_sequence: null, + usage: { input_tokens: tokensIn, output_tokens: tokensOut }, + } +} + +function anthropicBadRequest(message, status = 400, request) { + return json( + { type: "error", error: { type: "invalid_request_error", message } }, + status, + {}, + request, + ) +} + +function anthropicInternalError(message, status = 500, request) { + return json( + { type: "error", error: { type: "api_error", message } }, + status, + {}, + request, + ) +} + +// --------------------------------------------------------------------------- +// Google Gemini API helpers +// --------------------------------------------------------------------------- + +export function normalizeGeminiContents(contents) { + if (!Array.isArray(contents)) return [] + return contents + .map((item) => { + const role = item.role === "model" ? "assistant" : (item.role ?? "user") + const content = Array.isArray(item.parts) + ? item.parts + .map((part) => (typeof part?.text === "string" ? part.text.trim() : "")) + .filter(Boolean) + .join("\n\n") + : "" + return { role, content } + }) + .filter((m) => m.content.length > 0) +} + +export function extractGeminiSystemInstruction(systemInstruction) { + if (!systemInstruction) return null + if (typeof systemInstruction === "string") return systemInstruction.trim() + if (Array.isArray(systemInstruction.parts)) { + return systemInstruction.parts + .map((part) => (typeof part?.text === "string" ? part.text.trim() : "")) + .filter(Boolean) + .join("\n\n") + } + return null +} + +export function mapFinishReasonToGemini(finish) { + if (!finish) return "STOP" + if (finish.includes("length")) return "MAX_TOKENS" + if (finish.includes("tool")) return "STOP" + return "STOP" +} + +function createGeminiResponse(content, finish, tokens) { + return { + candidates: [ + { + content: { role: "model", parts: [{ text: content }] }, + finishReason: mapFinishReasonToGemini(finish), + index: 0, + }, + ], + usageMetadata: { + promptTokenCount: tokens?.input ?? 0, + candidatesTokenCount: tokens?.output ?? 0, + totalTokenCount: (tokens?.input ?? 0) + (tokens?.output ?? 0), + }, + } +} + +function geminiModelFromPath(pathname) { + // Matches /v1beta/models/some-model:generateContent or :streamGenerateContent + const match = pathname.match(/^\/v1beta\/models\/([^/:]+)(?::(?:generate|stream)(?:Content|GenerateContent))?$/) + return match ? match[1] : null +} + export function createProxyFetchHandler(client) { return async (request) => { const url = new URL(request.url) @@ -890,6 +1014,250 @@ export function createProxyFetchHandler(client) { } } + // ----------------------------------------------------------------------- + // Anthropic Messages API POST /v1/messages + // ----------------------------------------------------------------------- + + if (request.method === "POST" && url.pathname === "/v1/messages") { + let body + try { + body = await request.json() + } catch { + return anthropicBadRequest("Request body must be valid JSON.", 400, request) + } + + if (!body.model) { + return anthropicBadRequest("The 'model' field is required.", 400, request) + } + + if (!Array.isArray(body.messages) || body.messages.length === 0) { + return anthropicBadRequest("The 'messages' field must contain at least one message.", 400, request) + } + + const messages = normalizeAnthropicMessages(body.messages) + if (messages.length === 0) { + return anthropicBadRequest("No text content was found in the supplied messages.", 400, request) + } + + // Prepend Anthropic top-level system string as a system message so buildSystemPrompt picks it up. + const allMessages = + typeof body.system === "string" && body.system.trim() + ? [{ role: "system", content: body.system.trim() }, ...messages] + : messages + + const system = buildSystemPrompt(allMessages, { + temperature: body.temperature, + max_tokens: body.max_tokens, + }) + + let model + try { + const providerOverride = request.headers.get("x-opencode-provider") + model = await resolveModel(client, body.model, providerOverride) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + await safeLog(client, "error", "Anthropic proxy call failed (model resolve)", { error: message, requestedModel: body.model }) + return anthropicBadRequest(message, 400, request) + } + + if (body.stream) { + const msgID = `msg_${crypto.randomUUID().replace(/-/g, "")}` + const queue = createSseQueue() + + function sseEvent(eventType, data) { + return `event: ${eventType}\ndata: ${JSON.stringify(data)}\n\n` + } + + async function* generateSse() { + queue.enqueue(sseEvent("message_start", { + type: "message_start", + message: { + id: msgID, + type: "message", + role: "assistant", + content: [], + model: model.id, + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 0, output_tokens: 0 }, + }, + })) + queue.enqueue(sseEvent("content_block_start", { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + })) + + const runPromise = executePromptStreaming( + client, + model, + messages, + system, + (delta) => { + queue.enqueue(sseEvent("content_block_delta", { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: delta }, + })) + }, + ) + .then((streamResult) => { + queue.enqueue(sseEvent("content_block_stop", { type: "content_block_stop", index: 0 })) + queue.enqueue(sseEvent("message_delta", { + type: "message_delta", + delta: { + stop_reason: mapFinishReasonToAnthropic(streamResult.finish), + stop_sequence: null, + }, + usage: { output_tokens: streamResult.tokens.output }, + })) + queue.enqueue(sseEvent("message_stop", { type: "message_stop" })) + }) + .catch(async (err) => { + const errMsg = err instanceof Error ? err.message : String(err) + await safeLog(client, "error", "Anthropic proxy streaming call failed", { error: errMsg, requestedModel: body.model }) + queue.enqueue(sseEvent("error", { type: "error", error: { type: "api_error", message: errMsg } })) + }) + .finally(() => { + queue.finish() + }) + + yield* queue.generateChunks() + await runPromise + } + + return sseResponse(corsHeaders(request), generateSse()) + } + + try { + const result = await executePrompt(client, body, model, messages, system) + return json(createAnthropicResponse(result, model), 200, {}, request) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + await safeLog(client, "error", "Anthropic proxy call failed", { error: message, requestedModel: body.model }) + return anthropicInternalError(message, 500, request) + } + } + + // ----------------------------------------------------------------------- + // Google Gemini API POST /v1beta/models/:model:generateContent (non-streaming) + // POST /v1beta/models/:model:streamGenerateContent (streaming) + // ----------------------------------------------------------------------- + + const isGeminiNonStream = request.method === "POST" && url.pathname.endsWith(":generateContent") + const isGeminiStream = request.method === "POST" && url.pathname.endsWith(":streamGenerateContent") + + if (isGeminiNonStream || isGeminiStream) { + const geminiModelName = geminiModelFromPath(url.pathname) + if (!geminiModelName) { + return badRequest("Could not extract model name from URL.", 400, request) + } + + let body + try { + body = await request.json() + } catch { + return badRequest("Request body must be valid JSON.", 400, request) + } + + if (!Array.isArray(body.contents) || body.contents.length === 0) { + return badRequest("The 'contents' field must contain at least one item.", 400, request) + } + + const messages = normalizeGeminiContents(body.contents) + if (messages.length === 0) { + return badRequest("No text content was found in the supplied contents.", 400, request) + } + + const systemText = extractGeminiSystemInstruction(body.systemInstruction) + const systemMessages = systemText ? [{ role: "system", content: systemText }, ...messages] : messages + const system = buildSystemPrompt(systemMessages, { + temperature: body.generationConfig?.temperature, + max_tokens: body.generationConfig?.maxOutputTokens, + }) + + let model + try { + const providerOverride = request.headers.get("x-opencode-provider") + model = await resolveModel(client, geminiModelName, providerOverride) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + await safeLog(client, "error", "Gemini proxy call failed (model resolve)", { error: message, requestedModel: geminiModelName }) + return badRequest(message, 400, request) + } + + if (isGeminiStream) { + const queue = createSseQueue() + + async function* generateNdJson() { + const runPromise = executePromptStreaming( + client, + model, + messages, + system, + (delta) => { + const chunk = JSON.stringify(createGeminiResponse(delta, null, null)) + queue.enqueue(chunk + "\n") + }, + ) + .then((streamResult) => { + const finalChunk = JSON.stringify( + createGeminiResponse("", streamResult.finish, streamResult.tokens), + ) + queue.enqueue(finalChunk + "\n") + }) + .catch(async (err) => { + const errMsg = err instanceof Error ? err.message : String(err) + await safeLog(client, "error", "Gemini proxy streaming call failed", { error: errMsg, requestedModel: geminiModelName }) + const errChunk = JSON.stringify({ error: { code: 500, message: errMsg, status: "INTERNAL" } }) + queue.enqueue(errChunk + "\n") + }) + .finally(() => { + queue.finish() + }) + + yield* queue.generateChunks() + await runPromise + } + + const encoder = new TextEncoder() + const body_ = new ReadableStream({ + async start(controller) { + try { + for await (const chunk of generateNdJson()) { + controller.enqueue(encoder.encode(chunk)) + } + } catch { + // errors surfaced via data + } finally { + controller.close() + } + }, + }) + + return new Response(body_, { + status: 200, + headers: { + "content-type": "application/json", + "cache-control": "no-cache", + connection: "keep-alive", + ...corsHeaders(request), + }, + }) + } + + try { + const result = await executePrompt(client, body, model, messages, system) + const finish = result.completion.data.info?.finish + const tokens = result.completion.data.info?.tokens + return json(createGeminiResponse(result.content, finish, tokens), 200, {}, request) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + await safeLog(client, "error", "Gemini proxy call failed", { error: message, requestedModel: geminiModelName }) + return badRequest(message, 500, request) + } + } + return text("Not found", 404, request) } } diff --git a/index.test.js b/index.test.js index f26b033..f663ea0 100644 --- a/index.test.js +++ b/index.test.js @@ -12,6 +12,11 @@ import { extractAssistantText, mapFinishReason, resolveModel, + normalizeAnthropicMessages, + mapFinishReasonToAnthropic, + normalizeGeminiContents, + extractGeminiSystemInstruction, + mapFinishReasonToGemini, } from "./index.js" // --------------------------------------------------------------------------- @@ -1131,3 +1136,553 @@ test("POST /v1/responses stream: true with session.error emits response.failed", const text = await response.text() assert.ok(text.includes("response.failed") || text.includes("Rate limit exceeded")) }) + +// --------------------------------------------------------------------------- +// Unit: normalizeAnthropicMessages +// --------------------------------------------------------------------------- +describe("normalizeAnthropicMessages", () => { + it("passes through string content unchanged", () => { + const input = [{ role: "user", content: "hello" }] + assert.deepEqual(normalizeAnthropicMessages(input), [{ role: "user", content: "hello" }]) + }) + + it("trims whitespace from string content", () => { + const input = [{ role: "user", content: " hi " }] + assert.deepEqual(normalizeAnthropicMessages(input), [{ role: "user", content: "hi" }]) + }) + + it("joins text blocks from array content", () => { + const input = [ + { + role: "user", + content: [ + { type: "text", text: "first" }, + { type: "text", text: "second" }, + ], + }, + ] + assert.deepEqual(normalizeAnthropicMessages(input), [{ role: "user", content: "first\n\nsecond" }]) + }) + + it("ignores non-text blocks in array content", () => { + const input = [ + { + role: "user", + content: [ + { type: "image", source: {} }, + { type: "text", text: "only this" }, + ], + }, + ] + assert.deepEqual(normalizeAnthropicMessages(input), [{ role: "user", content: "only this" }]) + }) + + it("drops messages with empty content", () => { + const input = [ + { role: "user", content: "" }, + { role: "assistant", content: "response" }, + ] + assert.deepEqual(normalizeAnthropicMessages(input), [{ role: "assistant", content: "response" }]) + }) +}) + +// --------------------------------------------------------------------------- +// Unit: mapFinishReasonToAnthropic +// --------------------------------------------------------------------------- +describe("mapFinishReasonToAnthropic", () => { + it("returns end_turn for undefined", () => { + assert.equal(mapFinishReasonToAnthropic(undefined), "end_turn") + }) + + it("returns end_turn for null", () => { + assert.equal(mapFinishReasonToAnthropic(null), "end_turn") + }) + + it("returns max_tokens when finish includes length", () => { + assert.equal(mapFinishReasonToAnthropic("max_length"), "max_tokens") + }) + + it("returns tool_use when finish includes tool", () => { + assert.equal(mapFinishReasonToAnthropic("tool_use"), "tool_use") + }) + + it("returns end_turn for unrecognised values", () => { + assert.equal(mapFinishReasonToAnthropic("stop"), "end_turn") + }) +}) + +// --------------------------------------------------------------------------- +// Unit: normalizeGeminiContents +// --------------------------------------------------------------------------- +describe("normalizeGeminiContents", () => { + it("returns empty array for non-array input", () => { + assert.deepEqual(normalizeGeminiContents(null), []) + assert.deepEqual(normalizeGeminiContents("string"), []) + }) + + it("converts user role and joins text parts", () => { + const contents = [{ role: "user", parts: [{ text: "hello" }] }] + assert.deepEqual(normalizeGeminiContents(contents), [{ role: "user", content: "hello" }]) + }) + + it("maps model role to assistant", () => { + const contents = [{ role: "model", parts: [{ text: "hi there" }] }] + assert.deepEqual(normalizeGeminiContents(contents), [{ role: "assistant", content: "hi there" }]) + }) + + it("joins multiple parts with double newline", () => { + const contents = [{ role: "user", parts: [{ text: "line one" }, { text: "line two" }] }] + assert.deepEqual(normalizeGeminiContents(contents), [{ role: "user", content: "line one\n\nline two" }]) + }) + + it("drops items with no text content", () => { + const contents = [ + { role: "user", parts: [{ text: "" }] }, + { role: "user", parts: [{ text: "kept" }] }, + ] + assert.deepEqual(normalizeGeminiContents(contents), [{ role: "user", content: "kept" }]) + }) +}) + +// --------------------------------------------------------------------------- +// Unit: extractGeminiSystemInstruction +// --------------------------------------------------------------------------- +describe("extractGeminiSystemInstruction", () => { + it("returns null for null/undefined input", () => { + assert.equal(extractGeminiSystemInstruction(null), null) + assert.equal(extractGeminiSystemInstruction(undefined), null) + }) + + it("returns trimmed string for string input", () => { + assert.equal(extractGeminiSystemInstruction(" be helpful "), "be helpful") + }) + + it("joins parts array", () => { + const si = { parts: [{ text: "be concise" }, { text: "and clear" }] } + assert.equal(extractGeminiSystemInstruction(si), "be concise\n\nand clear") + }) + + it("returns null for object without parts", () => { + assert.equal(extractGeminiSystemInstruction({ role: "system" }), null) + }) +}) + +// --------------------------------------------------------------------------- +// Unit: mapFinishReasonToGemini +// --------------------------------------------------------------------------- +describe("mapFinishReasonToGemini", () => { + it("returns STOP for undefined", () => { + assert.equal(mapFinishReasonToGemini(undefined), "STOP") + }) + + it("returns MAX_TOKENS when finish includes length", () => { + assert.equal(mapFinishReasonToGemini("max_length"), "MAX_TOKENS") + }) + + it("returns STOP for tool_use", () => { + assert.equal(mapFinishReasonToGemini("tool_use"), "STOP") + }) + + it("returns STOP for end_turn", () => { + assert.equal(mapFinishReasonToGemini("end_turn"), "STOP") + }) +}) + +// --------------------------------------------------------------------------- +// Integration: POST /v1/messages (Anthropic Messages API) +// --------------------------------------------------------------------------- + +function createAnthropicClient(responseContent = "Hello from Anthropic.") { + return { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [ + { + id: "anthropic", + models: { "claude-3-5-sonnet": { id: "claude-3-5-sonnet", name: "Claude 3.5 Sonnet" } }, + }, + ], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-ant-1" } }), + prompt: async () => ({ + data: { + parts: [{ type: "text", text: responseContent }], + info: { tokens: { input: 15, output: 10, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "end_turn" }, + }, + }), + }, + } +} + +test("POST /v1/messages returns a well-formed Anthropic response", async () => { + const handler = createProxyFetchHandler(createAnthropicClient("Hi there!")) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "anthropic/claude-3-5-sonnet", + max_tokens: 1024, + messages: [{ role: "user", content: "Say hello." }], + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.equal(body.type, "message") + assert.equal(body.role, "assistant") + assert.ok(body.id.startsWith("msg_")) + assert.ok(Array.isArray(body.content)) + assert.equal(body.content[0].type, "text") + assert.equal(body.content[0].text, "Hi there!") + assert.equal(body.stop_reason, "end_turn") + assert.equal(body.usage.input_tokens, 15) + assert.equal(body.usage.output_tokens, 10) +}) + +test("POST /v1/messages system string is included in prompt", async () => { + let capturedSystem = null + const client = { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [{ id: "anthropic", models: { "claude-3-5-sonnet": { id: "claude-3-5-sonnet" } } }], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-ant-sys" } }), + prompt: async ({ body }) => { + capturedSystem = body.system + return { + data: { + parts: [{ type: "text", text: "ok" }], + info: { tokens: { input: 1, output: 1, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "end_turn" }, + }, + } + }, + }, + } + + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "anthropic/claude-3-5-sonnet", + system: "You are a pirate.", + messages: [{ role: "user", content: "Hello." }], + }), + }) + + await handler(request) + assert.ok(capturedSystem?.includes("You are a pirate.")) +}) + +test("POST /v1/messages missing model returns Anthropic error format", async () => { + const handler = createProxyFetchHandler(createAnthropicClient()) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [{ role: "user", content: "hi" }] }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.equal(body.type, "error") + assert.ok(body.error.type === "invalid_request_error") + assert.ok(body.error.message.includes("model")) +}) + +test("POST /v1/messages missing messages returns 400", async () => { + const handler = createProxyFetchHandler(createAnthropicClient()) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-3-5-sonnet" }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.equal(body.type, "error") +}) + +test("POST /v1/messages malformed JSON returns 400", async () => { + const handler = createProxyFetchHandler(createAnthropicClient()) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{ bad json", + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.equal(body.type, "error") +}) + +test("POST /v1/messages stream: true returns Anthropic SSE events", async () => { + const events = [ + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: "Hello", + }, + }, + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: " world", + }, + }, + { type: "session.idle", properties: { sessionID: "sess-123" } }, + ] + + const handler = createProxyFetchHandler(createStreamingClient(events)) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + messages: [{ role: "user", content: "hi" }], + }), + }) + + const response = await handler(request) + + assert.equal(response.status, 200) + assert.ok(response.headers.get("content-type")?.includes("text/event-stream")) + + const text = await response.text() + assert.ok(text.includes("message_start")) + assert.ok(text.includes("content_block_start")) + assert.ok(text.includes("content_block_delta")) + assert.ok(text.includes("Hello")) + assert.ok(text.includes(" world")) + assert.ok(text.includes("message_stop")) +}) + +test("POST /v1/messages stream: true with session.error emits SSE error event", async () => { + const events = [ + { + type: "session.error", + properties: { + sessionID: "sess-123", + error: { message: "Model overloaded" }, + }, + }, + { type: "session.idle", properties: { sessionID: "sess-123" } }, + ] + + const handler = createProxyFetchHandler(createStreamingClient(events)) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + messages: [{ role: "user", content: "hi" }], + }), + }) + + const response = await handler(request) + assert.equal(response.status, 200) + + const text = await response.text() + assert.ok(text.includes("error") || text.includes("Model overloaded")) +}) + +// --------------------------------------------------------------------------- +// Integration: POST /v1beta/models/:model:generateContent (Gemini API) +// --------------------------------------------------------------------------- + +function createGeminiClient(responseContent = "Hello from Gemini.") { + return { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [ + { + id: "google", + models: { "gemini-2.0-flash": { id: "gemini-2.0-flash", name: "Gemini 2.0 Flash" } }, + }, + ], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-gem-1" } }), + prompt: async () => ({ + data: { + parts: [{ type: "text", text: responseContent }], + info: { tokens: { input: 12, output: 7, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "end_turn" }, + }, + }), + }, + } +} + +test("POST /v1beta/models/gemini-2.0-flash:generateContent returns Gemini response", async () => { + const handler = createProxyFetchHandler(createGeminiClient("Gemini says hi!")) + const request = new Request("http://127.0.0.1:4010/v1beta/models/gemini-2.0-flash:generateContent", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "Say hi." }] }], + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.ok(Array.isArray(body.candidates)) + assert.equal(body.candidates[0].content.role, "model") + assert.equal(body.candidates[0].content.parts[0].text, "Gemini says hi!") + assert.equal(body.candidates[0].finishReason, "STOP") + assert.equal(body.usageMetadata.promptTokenCount, 12) + assert.equal(body.usageMetadata.candidatesTokenCount, 7) + assert.equal(body.usageMetadata.totalTokenCount, 19) +}) + +test("POST /v1beta/models/:model:generateContent systemInstruction is included", async () => { + let capturedSystem = null + const client = { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [{ id: "google", models: { "gemini-2.0-flash": { id: "gemini-2.0-flash" } } }], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-gem-sys" } }), + prompt: async ({ body }) => { + capturedSystem = body.system + return { + data: { + parts: [{ type: "text", text: "ok" }], + info: { tokens: { input: 1, output: 1, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "end_turn" }, + }, + } + }, + }, + } + + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1beta/models/gemini-2.0-flash:generateContent", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "Hello." }] }], + systemInstruction: { parts: [{ text: "You are a helpful assistant." }] }, + }), + }) + + await handler(request) + assert.ok(capturedSystem?.includes("You are a helpful assistant.")) +}) + +test("POST /v1beta/models/:model:generateContent missing contents returns 400", async () => { + const handler = createProxyFetchHandler(createGeminiClient()) + const request = new Request("http://127.0.0.1:4010/v1beta/models/gemini-2.0-flash:generateContent", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ generationConfig: { maxOutputTokens: 100 } }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 400) + assert.ok(body.error.message.includes("contents")) +}) + +test("POST /v1beta/models/:model:generateContent malformed JSON returns 400", async () => { + const handler = createProxyFetchHandler(createGeminiClient()) + const request = new Request("http://127.0.0.1:4010/v1beta/models/gemini-2.0-flash:generateContent", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{ not json", + }) + + const response = await handler(request) + + assert.equal(response.status, 400) +}) + +test("POST /v1beta/models/:model:streamGenerateContent returns NDJSON stream", async () => { + const events = [ + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: "Gem", + }, + }, + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: "ini", + }, + }, + { type: "session.idle", properties: { sessionID: "sess-123" } }, + ] + + // Use streaming client but swap provider to google + const streamingClient = createStreamingClient(events) + streamingClient.config = { + providers: async () => ({ + data: { + providers: [ + { id: "google", models: { "gemini-2.0-flash": { id: "gemini-2.0-flash", name: "Gemini 2.0 Flash" } } }, + ], + }, + }), + } + + const handler = createProxyFetchHandler(streamingClient) + const request = new Request( + "http://127.0.0.1:4010/v1beta/models/gemini-2.0-flash:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "Stream this." }] }], + }), + }, + ) + + const response = await handler(request) + + assert.equal(response.status, 200) + assert.ok(response.headers.get("content-type")?.includes("application/json")) + + const text = await response.text() + // Should contain NDJSON lines with candidates + assert.ok(text.includes("candidates")) + assert.ok(text.includes("Gem")) + assert.ok(text.includes("ini")) +}) From 0e7a80cbebb3f4ca29921b685899d1cb59c75832 Mon Sep 17 00:00:00 2001 From: KochC Date: Fri, 27 Mar 2026 17:19:31 +0100 Subject: [PATCH 05/11] docs: rewrite README and expand package keywords for discoverability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Lead with value proposition, ASCII diagram, and feature table - Quickstart reduced to 4 steps; works in under 60 seconds - SDK examples for OpenAI, Anthropic, Gemini (JS+Python), LangChain - UI integration guides: Open WebUI, Chatbox, Continue, Zed - Reference section kept concise; full prose docs moved inline - package.json: sharper description, 20 keywords covering all search terms (openai-compatible, anthropic, gemini, ollama, langchain, open-webui, llm-proxy, ai-gateway, local-llm, github-copilot, model-router, …) --- README.md | 350 ++++++++++++++++++++++++++++++++++----------------- package.json | 19 ++- 2 files changed, 255 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index f85a06f..2077d16 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,53 @@ # opencode-llm-proxy -An [OpenCode](https://opencode.ai) plugin that starts a local HTTP server backed by your OpenCode providers, with support for multiple LLM API formats: +[![npm](https://img.shields.io/npm/v/opencode-llm-proxy)](https://www.npmjs.com/package/opencode-llm-proxy) +[![npm downloads](https://img.shields.io/npm/dm/opencode-llm-proxy)](https://www.npmjs.com/package/opencode-llm-proxy) +[![CI](https://github.com/KochC/opencode-llm-proxy/actions/workflows/ci.yml/badge.svg)](https://github.com/KochC/opencode-llm-proxy/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -- **OpenAI** Chat Completions (`POST /v1/chat/completions`) and Responses (`POST /v1/responses`) -- **Anthropic** Messages API (`POST /v1/messages`) -- **Google Gemini** API (`POST /v1beta/models/:model:generateContent`) +**One local endpoint. Every model you have access to. Any API format.** -Any tool or SDK that targets one of these APIs can point at the proxy without code changes. +opencode-llm-proxy is an [OpenCode](https://opencode.ai) plugin that starts a local HTTP server on `http://127.0.0.1:4010`. It translates between the API format your tool speaks and whichever LLM provider OpenCode has configured — so you never reconfigure the same models twice. + +``` +Your tool (OpenAI / Anthropic / Gemini SDK) + │ + ▼ http://127.0.0.1:4010 + opencode-llm-proxy + │ + ▼ OpenCode SDK + GitHub Copilot · Anthropic · Gemini · Ollama · OpenRouter · Bedrock · … +``` + +**Supported API formats — all with streaming:** + +| Format | Endpoint | +|---|---| +| OpenAI Chat Completions | `POST /v1/chat/completions` | +| OpenAI Responses API | `POST /v1/responses` | +| Anthropic Messages API | `POST /v1/messages` | +| Google Gemini | `POST /v1beta/models/:model:generateContent` | + +--- + +## Why + +Most LLM tools speak exactly one API dialect. OpenCode already manages connections to every provider you use. This proxy bridges the two — your tools keep working as-is, and you change which model they use in one place. + +**Common situations it solves:** + +- You have a **GitHub Copilot** subscription. Open WebUI, Chatbox, or a VS Code extension only accepts an OpenAI-compatible URL. Point them at the proxy — done. +- You run **Ollama** locally. Your Python scripts use the OpenAI SDK. Set `base_url` to the proxy and use your Ollama model IDs directly. +- You want to **swap models without code changes**. Your app talks to the proxy; you change the model in OpenCode config. +- You want to **share your models on a LAN**. Expose the proxy on `0.0.0.0` and give teammates the URL. +- You use the **Anthropic SDK** but want to route through GitHub Copilot or Bedrock. No code change in the SDK — just point it at the proxy. + +--- ## Quickstart ```bash -# 1. Install the npm package npm install opencode-llm-proxy - -# 2. Register the plugin in your opencode.json -# (or use one of the manual install methods below) ``` Add to `opencode.json`: @@ -26,11 +58,10 @@ Add to `opencode.json`: } ``` -Then start OpenCode — the proxy starts automatically: +Start OpenCode — the proxy starts automatically: ```bash opencode -# Proxy is now listening on http://127.0.0.1:4010 ``` Send a request: @@ -44,15 +75,17 @@ curl http://127.0.0.1:4010/v1/chat/completions \ }' ``` +--- + ## Install -### As an npm plugin (recommended) +### npm plugin (recommended) ```bash npm install opencode-llm-proxy ``` -Add to `opencode.json`: +Add to your global `~/.config/opencode/opencode.json` (works everywhere) or a project-level `opencode.json`: ```json { @@ -60,170 +93,263 @@ Add to `opencode.json`: } ``` -### As a global OpenCode plugin +### Copy the file -Copy `index.js` to your global plugin directory: +**Global** — loaded for every OpenCode session: ```bash -cp index.js ~/.config/opencode/plugins/openai-proxy.js +curl -o ~/.config/opencode/plugins/llm-proxy.js \ + https://raw.githubusercontent.com/KochC/opencode-llm-proxy/main/index.js ``` -The plugin is loaded automatically every time OpenCode starts. - -### As a project plugin - -Copy `index.js` to your project's plugin directory: +**Per-project** — loaded only in this directory: ```bash -cp index.js .opencode/plugins/openai-proxy.js +mkdir -p .opencode/plugins +curl -o .opencode/plugins/llm-proxy.js \ + https://raw.githubusercontent.com/KochC/opencode-llm-proxy/main/index.js ``` -## Usage +--- -Start OpenCode normally. The proxy server starts automatically in the background: +## Configuration -``` +| Variable | Default | Description | +|---|---|---| +| `OPENCODE_LLM_PROXY_HOST` | `127.0.0.1` | Bind address. `0.0.0.0` to expose on LAN or Docker. | +| `OPENCODE_LLM_PROXY_PORT` | `4010` | TCP port. | +| `OPENCODE_LLM_PROXY_TOKEN` | _(unset)_ | Bearer token required on every request. Unset = no auth. | +| `OPENCODE_LLM_PROXY_CORS_ORIGIN` | `*` | `Access-Control-Allow-Origin` value for browser clients. | + +```bash +OPENCODE_LLM_PROXY_HOST=0.0.0.0 \ +OPENCODE_LLM_PROXY_TOKEN=my-secret \ opencode ``` -The server listens on `http://127.0.0.1:4010` by default. +--- -### List available models +## Using with SDKs and tools -```bash -curl http://127.0.0.1:4010/v1/models -``` +### OpenAI SDK (JS/TS) -Returns all models from all providers configured in your OpenCode setup (e.g. `github-copilot/claude-sonnet-4.6`, `ollama/qwen3.5:9b`, etc.). +```javascript +import OpenAI from "openai" -### OpenAI Chat Completions +const client = new OpenAI({ + baseURL: "http://127.0.0.1:4010/v1", + apiKey: "unused", +}) -```bash -curl http://127.0.0.1:4010/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "github-copilot/claude-sonnet-4.6", - "messages": [ - {"role": "user", "content": "Write a haiku about OpenCode."} - ] - }' +const response = await client.chat.completions.create({ + model: "github-copilot/claude-sonnet-4.6", + messages: [{ role: "user", content: "Explain recursion." }], +}) ``` -Use the fully-qualified `provider/model` ID from `GET /v1/models`. Supports `"stream": true` for SSE streaming. +### OpenAI SDK (Python) -### OpenAI Responses API +```python +from openai import OpenAI -```bash -curl http://127.0.0.1:4010/v1/responses \ - -H "Content-Type: application/json" \ - -d '{ - "model": "github-copilot/claude-sonnet-4.6", - "input": [{"role": "user", "content": "Hello"}] - }' +client = OpenAI(base_url="http://127.0.0.1:4010/v1", api_key="unused") + +response = client.chat.completions.create( + model="ollama/qwen2.5-coder", + messages=[{"role": "user", "content": "Write a Python function to reverse a string."}], +) +print(response.choices[0].message.content) ``` -Supports `"stream": true` for SSE streaming. +### Anthropic SDK (Python) -### Anthropic Messages API +```python +import anthropic -Point the Anthropic SDK (or any client) at this proxy: +client = anthropic.Anthropic( + base_url="http://127.0.0.1:4010", + api_key="unused", +) -```bash -curl http://127.0.0.1:4010/v1/messages \ - -H "Content-Type: application/json" \ - -d '{ - "model": "anthropic/claude-3-5-sonnet", - "max_tokens": 1024, - "system": "You are a helpful assistant.", - "messages": [{"role": "user", "content": "Hello!"}] - }' +message = client.messages.create( + model="anthropic/claude-3-5-sonnet", + max_tokens=1024, + messages=[{"role": "user", "content": "What is the Pythagorean theorem?"}], +) +print(message.content[0].text) ``` -Supports `"stream": true` for SSE streaming with standard Anthropic streaming events (`message_start`, `content_block_delta`, `message_stop`, etc.). +### Anthropic SDK (JS/TS) -To point the official Anthropic SDK at this proxy: - -```js +```javascript import Anthropic from "@anthropic-ai/sdk" const client = new Anthropic({ baseURL: "http://127.0.0.1:4010", - apiKey: "unused", // or your OPENCODE_LLM_PROXY_TOKEN + apiKey: "unused", +}) + +const message = await client.messages.create({ + model: "anthropic/claude-opus-4", + max_tokens: 1024, + messages: [{ role: "user", content: "Explain async/await." }], }) ``` -### Google Gemini API +### Google Generative AI SDK (JS/TS) -```bash -# Non-streaming -curl http://127.0.0.1:4010/v1beta/models/google/gemini-2.0-flash:generateContent \ - -H "Content-Type: application/json" \ - -d '{ - "contents": [{"role": "user", "parts": [{"text": "Hello!"}]}] - }' +```javascript +import { GoogleGenerativeAI } from "@google/generative-ai" -# Streaming (newline-delimited JSON) -curl http://127.0.0.1:4010/v1beta/models/google/gemini-2.0-flash:streamGenerateContent \ - -H "Content-Type: application/json" \ - -d '{ - "contents": [{"role": "user", "parts": [{"text": "Hello!"}]}] - }' +const genAI = new GoogleGenerativeAI("unused", { + baseUrl: "http://127.0.0.1:4010", +}) + +const model = genAI.getGenerativeModel({ model: "google/gemini-2.0-flash" }) +const result = await model.generateContent("What is machine learning?") +console.log(result.response.text()) ``` -The model name in the URL path is resolved the same way as other endpoints (use `provider/model` or a bare model ID if unambiguous). +### LangChain (Python) -To point the Google Generative AI SDK at this proxy, set the `baseUrl` option to `http://127.0.0.1:4010`. +```python +from langchain_openai import ChatOpenAI -## Selecting a provider +llm = ChatOpenAI( + model="anthropic/claude-3-5-sonnet", + openai_api_base="http://127.0.0.1:4010/v1", + openai_api_key="unused", +) -All endpoints accept an optional `x-opencode-provider` header to force a specific provider when the model ID is ambiguous: +response = llm.invoke("What are the SOLID principles?") +print(response.content) +``` -```bash -curl http://127.0.0.1:4010/v1/chat/completions \ - -H "x-opencode-provider: anthropic" \ - -H "Content-Type: application/json" \ - -d '{"model": "claude-3-5-sonnet", "messages": [...]}' +### Open WebUI + +1. Settings → Connections → OpenAI API +2. Set **API Base URL** to `http://127.0.0.1:4010/v1` +3. Leave API Key blank (or set to your `OPENCODE_LLM_PROXY_TOKEN`) +4. Save — all your OpenCode models appear in the model picker + +> Running Open WebUI in Docker? Use `http://host.docker.internal:4010/v1` and set `OPENCODE_LLM_PROXY_HOST=0.0.0.0`. + +### Chatbox + +Settings → AI Provider → OpenAI API → set **API Host** to `http://127.0.0.1:4010`. + +### Continue (VS Code / JetBrains) + +In `~/.continue/config.json`: + +```json +{ + "models": [ + { + "title": "Claude via OpenCode", + "provider": "openai", + "model": "anthropic/claude-3-5-sonnet", + "apiBase": "http://127.0.0.1:4010/v1", + "apiKey": "unused" + } + ] +} ``` -## Configuration +### Zed -All configuration is done through environment variables. No configuration file is needed. +In `~/.config/zed/settings.json`: -| Variable | Type | Default | Description | -|---|---|---|---| -| `OPENCODE_LLM_PROXY_HOST` | string | `127.0.0.1` | Bind address. Set to `0.0.0.0` to expose on LAN. | -| `OPENCODE_LLM_PROXY_PORT` | integer | `4010` | TCP port the proxy listens on. | -| `OPENCODE_LLM_PROXY_TOKEN` | string | _(unset)_ | Optional bearer token. When set, every request must include `Authorization: Bearer `. Unset means no authentication required. | -| `OPENCODE_LLM_PROXY_CORS_ORIGIN` | string | `*` | Value of the `Access-Control-Allow-Origin` response header. Use a specific origin (e.g. `https://app.example.com`) when browser clients send credentials. | +```json +{ + "language_models": { + "openai": { + "api_url": "http://127.0.0.1:4010/v1", + "available_models": [ + { + "name": "github-copilot/claude-sonnet-4.6", + "display_name": "Claude (OpenCode)", + "max_tokens": 8096 + } + ] + } + } +} +``` -The proxy adds CORS headers to all responses and handles `OPTIONS` preflight requests automatically. +--- -### LAN example +## Finding model IDs ```bash -export OPENCODE_LLM_PROXY_HOST=0.0.0.0 -export OPENCODE_LLM_PROXY_PORT=4010 -export OPENCODE_LLM_PROXY_TOKEN=my-secret-token -opencode +curl http://127.0.0.1:4010/v1/models | jq '.data[].id' +# "github-copilot/claude-sonnet-4.6" +# "anthropic/claude-3-5-sonnet" +# "ollama/qwen2.5-coder" +# ... ``` -Then from another machine: +Use `provider/model` for clarity. Bare model IDs (e.g. `gpt-4o`) work if unambiguous across your providers. -```bash -curl http://:4010/v1/models \ - -H "Authorization: Bearer my-secret-token" +To force a specific provider without changing the model string, add: + +``` +x-opencode-provider: anthropic +``` + +--- + +## API reference + +### GET /health +```json +{ "healthy": true, "service": "opencode-openai-proxy" } ``` +### GET /v1/models +Returns all models from all configured providers in OpenAI list format. + +### POST /v1/chat/completions +OpenAI Chat Completions. Required fields: `model`, `messages`. Optional: `stream`, `temperature`, `max_tokens`. + +### POST /v1/responses +OpenAI Responses API. Required fields: `model`, `input`. Optional: `instructions`, `stream`, `max_output_tokens`. + +### POST /v1/messages +Anthropic Messages API. Required fields: `model`, `messages`. Optional: `system`, `max_tokens`, `stream`. + +Errors are returned in Anthropic format: `{ "type": "error", "error": { "type": "...", "message": "..." } }`. + +### POST /v1beta/models/:model:generateContent +Google Gemini non-streaming. Model name in URL path. Required field: `contents`. Optional: `systemInstruction`, `generationConfig`. + +### POST /v1beta/models/:model:streamGenerateContent +Same as above, returns newline-delimited JSON stream. + +--- + ## How it works -The plugin hooks into OpenCode at startup and spawns a Bun HTTP server. Incoming requests (in OpenAI, Anthropic, or Gemini format) are translated into OpenCode SDK calls (`client.session.create` + `client.session.prompt`), routed through whichever provider/model is requested, and the response is returned in the matching API format. +Each request: + +1. Is authenticated if `OPENCODE_LLM_PROXY_TOKEN` is set +2. Has its model resolved — `provider/model`, bare model ID, or Gemini URL path +3. Creates a temporary OpenCode session (visible in the session list) +4. Sends the prompt via `client.session.prompt` / `client.session.promptAsync` +5. Returns the response in the same format as the request -Each request creates a temporary OpenCode session, so prompts and responses appear in the OpenCode session list. +Streaming uses OpenCode's `client.event.subscribe()` SSE stream. Text deltas are forwarded in real time. + +--- ## Limitations -- Tool/function calling is not forwarded; all built-in OpenCode tools are disabled for proxy sessions. -- Only text content is handled; image and file inputs are ignored. +- Text only — image, audio, and file inputs are ignored +- No tool/function calling — all OpenCode tools are disabled for proxy sessions +- No cross-request session state — send full conversation history on every request +- Temperature and max tokens are advisory (passed as system prompt hints) + +--- ## License diff --git a/package.json b/package.json index e872da7..30bf2d9 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "opencode-llm-proxy", "version": "1.6.0", - "description": "OpenCode plugin that exposes an OpenAI-compatible HTTP proxy backed by any LLM provider configured in OpenCode", + "description": "Local AI gateway for OpenCode — use any model via OpenAI, Anthropic, or Gemini API format", "main": "index.js", "type": "module", "engines": { @@ -16,8 +16,23 @@ "opencode", "opencode-plugin", "openai", + "openai-compatible", + "anthropic", + "gemini", + "ollama", "proxy", - "llm" + "llm", + "ai", + "gateway", + "local-llm", + "github-copilot", + "langchain", + "open-webui", + "llm-proxy", + "ai-gateway", + "model-router", + "openrouter", + "bedrock" ], "author": "KochC", "license": "MIT", From a4a8688e64582c557d2b78cfa41529cf8aa92c7c Mon Sep 17 00:00:00 2001 From: KochC Date: Fri, 27 Mar 2026 17:25:13 +0100 Subject: [PATCH 06/11] fix: remove pretty-printing from JSON responses to reduce payload size --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 227f01e..d6ac27c 100644 --- a/index.js +++ b/index.js @@ -30,7 +30,7 @@ function corsHeaders(request) { } function json(data, status = 200, headers = {}, request) { - return new Response(JSON.stringify(data, null, 2), { + return new Response(JSON.stringify(data), { status, headers: { "content-type": "application/json; charset=utf-8", From d5b14e63bdd6220c41335043de2878f074df642a Mon Sep 17 00:00:00 2001 From: KochC Date: Fri, 27 Mar 2026 17:29:08 +0100 Subject: [PATCH 07/11] fix: reflect request Origin header in CORS allow-origin when a specific origin is configured --- index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index d6ac27c..e3532e9 100644 --- a/index.js +++ b/index.js @@ -12,7 +12,8 @@ function corsHeaders(request) { const requestedHeaders = request?.headers.get("access-control-request-headers") const requestedMethod = request?.headers.get("access-control-request-method") const requestedPrivateNetwork = request?.headers.get("access-control-request-private-network") - const allowOrigin = configuredOrigin === "*" ? "*" : configuredOrigin + const requestOrigin = request?.headers.get("origin") ?? "" + const allowOrigin = configuredOrigin === "*" ? "*" : (requestOrigin === configuredOrigin ? requestOrigin : configuredOrigin) const headers = { vary: "origin, access-control-request-method, access-control-request-headers", From a0cfc7c39d0fe6d126f981314e49b31b451d718a Mon Sep 17 00:00:00 2001 From: KochC Date: Fri, 27 Mar 2026 17:45:30 +0100 Subject: [PATCH 08/11] docs: add LICENSE, CONTRIBUTING guide, and GitHub issue templates --- .github/ISSUE_TEMPLATE/bug_report.yml | 65 +++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.yml | 43 ++++++++++++ CONTRIBUTING.md | 82 ++++++++++++++++++++++ LICENSE | 21 ++++++ 4 files changed, 211 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..8beda4d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,65 @@ +name: Bug report +description: Something isn't working as expected +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to report a bug. Please fill in as much detail as you can. + + - type: textarea + id: description + attributes: + label: What happened? + description: A clear description of the bug. + validations: + required: true + + - type: textarea + id: reproduction + attributes: + label: Steps to reproduce + description: How do we reproduce the issue? + placeholder: | + 1. Start opencode with the plugin loaded + 2. Send a request to POST /v1/chat/completions with ... + 3. See error + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behaviour + description: What did you expect to happen? + validations: + required: true + + - type: textarea + id: request + attributes: + label: Request / response (if applicable) + description: Paste the curl command or request body and the response you received. + render: bash + + - type: input + id: version + attributes: + label: opencode-llm-proxy version + placeholder: "e.g. 1.6.1" + validations: + required: true + + - type: input + id: runtime + attributes: + label: Runtime and OS + placeholder: "e.g. Node.js 22, macOS 14 / Bun 1.2, Ubuntu 24.04" + validations: + required: true + + - type: input + id: provider + attributes: + label: Provider / model + placeholder: "e.g. github-copilot/claude-sonnet-4.6" diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..671cd64 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,43 @@ +name: Feature request +description: Suggest a new feature or improvement +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for the suggestion! Please describe the use case clearly so we can understand what you need. + + - type: textarea + id: problem + attributes: + label: What problem does this solve? + description: Describe the situation where this would be useful. + placeholder: "e.g. I use the Vercel AI SDK and currently have to..." + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Proposed solution + description: What would you like to see added or changed? + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives you've considered + description: Any workarounds you're using today? + + - type: dropdown + id: api_format + attributes: + label: Which API format does this relate to? (if any) + options: + - OpenAI Chat Completions + - OpenAI Responses API + - Anthropic Messages API + - Google Gemini + - All / general + - Not API-format specific diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..575fbeb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,82 @@ +# Contributing + +Thanks for your interest in contributing to opencode-llm-proxy. + +## Getting started + +```bash +git clone https://github.com/KochC/opencode-llm-proxy.git +cd opencode-llm-proxy +npm install +``` + +Run the tests: + +```bash +npm test +``` + +Run the linter: + +```bash +npm run lint +``` + +## How to contribute + +### Reporting bugs + +Open a [bug report](https://github.com/KochC/opencode-llm-proxy/issues/new?template=bug_report.yml). Include: + +- What you did +- What you expected +- What actually happened +- Your Node.js / Bun version and OS + +### Suggesting features + +Open a [feature request](https://github.com/KochC/opencode-llm-proxy/issues/new?template=feature_request.yml) describing the use case. + +### Submitting a pull request + +1. Fork the repo and create a branch from `dev` (not `main`) +2. Make your changes +3. Add or update tests in `index.test.js` — all 112+ tests must pass +4. Lint passes: `npm run lint` +5. Commit using [Conventional Commits](https://www.conventionalcommits.org/): + - `fix:` for bug fixes (triggers a patch release) + - `feat:` for new features (triggers a minor release) + - `docs:` / `chore:` / `test:` for everything else (no release) +6. Open a PR against the `dev` branch + +## Branch model + +``` +dev ──► main ──► npm (via Release Please) +``` + +- All work goes on `dev` +- `main` is release-only — only Release Please PRs merge directly there +- Do not open PRs against `main` + +## Tests + +Tests use the Node.js built-in test runner — no external framework needed. + +```bash +node --test # run once +node --test --watch # watch mode +node --test --experimental-test-coverage # with coverage +``` + +Tests mock the OpenCode SDK client entirely — no real LLM calls are made. + +## Code style + +ESLint enforces style. Run `npm run lint` before pushing. The config is in `eslint.config.js`. + +Key conventions in the codebase: + +- Pure functions are exported for testability (`normalizeMessages`, `buildPrompt`, etc.) +- Each API format (OpenAI, Anthropic, Gemini) has its own section in `index.js` +- Error responses mirror the format of the target API (OpenAI errors for `/v1/*`, Anthropic errors for `/v1/messages`, etc.) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bbf87e4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 KochC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 2eb182f9114d006ededd4a8d2556cb4f1917ef70 Mon Sep 17 00:00:00 2001 From: mikemolinet Date: Sat, 4 Jul 2026 14:06:56 -0700 Subject: [PATCH 09/11] fix: accept Anthropic system field as content-block array (#47) The Anthropic Messages API accepts the top-level `system` field as either a string OR an array of content blocks (per https://docs.anthropic.com/en/api/messages). The /v1/messages handler at index.js:1044-1047 only checks `typeof body.system === "string"` and silently drops the array form. Clients that follow the spec see their system prompt ignored by the proxy. Add and export a `normalizeAnthropicSystem` helper that accepts either form: for the array form, concatenates `type: "text"` content blocks (skipping falsy entries, non-text types, and non-string texts); returns null when no usable text is present so the call site can skip adding an empty system message. Use it at the call site in place of the inline string check. Adds 3 regression tests in index.test.js covering: - array-form system reaches buildSystemPrompt (discriminating) - multi-block text arrays are concatenated - helper edge cases (null/undefined, empty strings, non-text blocks, non-string/non-array inputs) Closes #46 --- README.md | 2 +- index.js | 28 ++++++++++--- index.test.js | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2077d16..099ca1a 100644 --- a/README.md +++ b/README.md @@ -316,7 +316,7 @@ OpenAI Chat Completions. Required fields: `model`, `messages`. Optional: `stream OpenAI Responses API. Required fields: `model`, `input`. Optional: `instructions`, `stream`, `max_output_tokens`. ### POST /v1/messages -Anthropic Messages API. Required fields: `model`, `messages`. Optional: `system`, `max_tokens`, `stream`. +Anthropic Messages API. Required fields: `model`, `messages`. Optional: `system` (string or array of `{type: "text", text: string}` content blocks), `max_tokens`, `stream`. Errors are returned in Anthropic format: `{ "type": "error", "error": { "type": "...", "message": "..." } }`. diff --git a/index.js b/index.js index e3532e9..9701fef 100644 --- a/index.js +++ b/index.js @@ -592,6 +592,22 @@ export function normalizeAnthropicMessages(messages) { .filter((message) => message.content.length > 0) } +export function normalizeAnthropicSystem(system) { + if (typeof system === "string") { + const trimmed = system.trim() + return trimmed || null + } + if (Array.isArray(system)) { + const text = system + .filter((block) => block && block.type === "text" && typeof block.text === "string") + .map((block) => block.text.trim()) + .filter(Boolean) + .join("\n\n") + return text || null + } + return null +} + export function mapFinishReasonToAnthropic(finish) { if (!finish) return "end_turn" if (finish.includes("length")) return "max_tokens" @@ -1040,11 +1056,13 @@ export function createProxyFetchHandler(client) { return anthropicBadRequest("No text content was found in the supplied messages.", 400, request) } - // Prepend Anthropic top-level system string as a system message so buildSystemPrompt picks it up. - const allMessages = - typeof body.system === "string" && body.system.trim() - ? [{ role: "system", content: body.system.trim() }, ...messages] - : messages + // Prepend Anthropic top-level `system` (string or array-of-content-blocks, + // per the Messages API spec) as a system message so buildSystemPrompt + // picks it up. + const systemText = normalizeAnthropicSystem(body.system) + const allMessages = systemText + ? [{ role: "system", content: systemText }, ...messages] + : messages const system = buildSystemPrompt(allMessages, { temperature: body.temperature, diff --git a/index.test.js b/index.test.js index f663ea0..f07c01d 100644 --- a/index.test.js +++ b/index.test.js @@ -14,6 +14,7 @@ import { resolveModel, normalizeAnthropicMessages, mapFinishReasonToAnthropic, + normalizeAnthropicSystem, normalizeGeminiContents, extractGeminiSystemInstruction, mapFinishReasonToGemini, @@ -1388,6 +1389,114 @@ test("POST /v1/messages system string is included in prompt", async () => { assert.ok(capturedSystem?.includes("You are a pirate.")) }) +test("POST /v1/messages system as content-block array is included in prompt", async () => { + let capturedSystem = null + const client = { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [{ id: "anthropic", models: { "claude-3-5-sonnet": { id: "claude-3-5-sonnet" } } }], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-ant-sys-arr" } }), + prompt: async ({ body }) => { + capturedSystem = body.system + return { + data: { + parts: [{ type: "text", text: "ok" }], + info: { tokens: { input: 1, output: 1, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "end_turn" }, + }, + } + }, + }, + } + + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "anthropic/claude-3-5-sonnet", + system: [{ type: "text", text: "You are a pirate." }], + messages: [{ role: "user", content: "Hello." }], + }), + }) + + await handler(request) + assert.ok(capturedSystem?.includes("You are a pirate.")) +}) + +test("POST /v1/messages system as multi-block array concatenates text", async () => { + let capturedSystem = null + const client = { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: [{ id: "anthropic", models: { "claude-3-5-sonnet": { id: "claude-3-5-sonnet" } } }], + }, + }), + }, + session: { + create: async () => ({ data: { id: "sess-ant-sys-multi" } }), + prompt: async ({ body }) => { + capturedSystem = body.system + return { + data: { + parts: [{ type: "text", text: "ok" }], + info: { tokens: { input: 1, output: 1, reasoning: 0, cache: { read: 0, write: 0 } }, finish: "end_turn" }, + }, + } + }, + }, + } + + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "anthropic/claude-3-5-sonnet", + system: [ + { type: "text", text: "Line one." }, + { type: "text", text: "Line two." }, + ], + messages: [{ role: "user", content: "Hello." }], + }), + }) + + await handler(request) + assert.ok(capturedSystem?.includes("Line one.")) + assert.ok(capturedSystem?.includes("Line two.")) +}) + +test("normalizeAnthropicSystem handles string, array, and edge cases", () => { + assert.equal(normalizeAnthropicSystem("hello"), "hello") + assert.equal(normalizeAnthropicSystem(" hi "), "hi") + assert.equal(normalizeAnthropicSystem(""), null) + assert.equal(normalizeAnthropicSystem(" "), null) + assert.equal(normalizeAnthropicSystem([{ type: "text", text: "a" }]), "a") + assert.equal( + normalizeAnthropicSystem([ + { type: "text", text: "a" }, + { type: "text", text: "b" }, + ]), + "a\n\nb", + ) + assert.equal(normalizeAnthropicSystem([{ type: "image", source: {} }]), null) + assert.equal(normalizeAnthropicSystem([]), null) + assert.equal(normalizeAnthropicSystem([{ type: "text", text: "" }]), null) + assert.equal(normalizeAnthropicSystem(undefined), null) + assert.equal(normalizeAnthropicSystem(null), null) + assert.equal(normalizeAnthropicSystem(42), null) + assert.equal(normalizeAnthropicSystem([null, { type: "text", text: "x" }]), "x") +}) + test("POST /v1/messages missing model returns Anthropic error format", async () => { const handler = createProxyFetchHandler(createAnthropicClient()) const request = new Request("http://127.0.0.1:4010/v1/messages", { From b7402cabf22720087158e52d217c64978775ae6b Mon Sep 17 00:00:00 2001 From: mikemolinet Date: Sat, 4 Jul 2026 14:07:04 -0700 Subject: [PATCH 10/11] fix: emit content_part.done and populate output_text.done.text per Responses API spec (#49) The /v1/responses streaming handler violates the OpenAI Responses API SSE lifecycle spec in two ways: 1. response.content_part.done is never emitted. Per the spec (https://platform.openai.com/docs/api-reference/responses-streaming), the event sequence for a text content part should be: content_part.added -> output_text.delta* -> output_text.done -> content_part.done -> output_item.done 2. response.output_text.done is emitted with text: "" instead of the accumulated output text. The spec requires the final content. Accumulate delta tokens in a local variable at the streaming call site, emit the missing response.content_part.done event with the accumulated text in part.text, and populate output_text.done.text with the same accumulated content. Gate the new content_part.done event on at least one delta having been received, keeping the content-part added/done lifecycle symmetric. Adds one regression test in index.test.js that asserts: - output_text.done.text equals the accumulated deltas - content_part.done event is present with part.text populated - correct ordering (output_text.done < content_part.done < output_item.done) Closes #48 --- index.js | 19 ++++++++++++- index.test.js | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 9701fef..a07678e 100644 --- a/index.js +++ b/index.js @@ -923,6 +923,9 @@ export function createProxyFetchHandler(client) { ) let partIndex = 0 + // Accumulate delta tokens so we can populate `text` on output_text.done and content_part.done per the + // OpenAI Responses API SSE spec (https://platform.openai.com/docs/api-reference/responses-streaming). + let accumulatedText = "" const runPromise = executePromptStreaming( client, model, @@ -941,6 +944,7 @@ export function createProxyFetchHandler(client) { ) partIndex++ } + accumulatedText += delta queue.enqueue( sseEvent("response.output_text.delta", { type: "response.output_text.delta", @@ -959,9 +963,22 @@ export function createProxyFetchHandler(client) { item_id: itemID, output_index: 0, content_index: 0, - text: "", + text: accumulatedText, }), ) + if (partIndex > 0) { + // Only emit content_part.done if content_part.added was emitted (i.e. at least one delta arrived). + // Keeps the content-part lifecycle symmetric per the OpenAI Responses API spec. + queue.enqueue( + sseEvent("response.content_part.done", { + type: "response.content_part.done", + item_id: itemID, + output_index: 0, + content_index: 0, + part: { type: "output_text", text: accumulatedText, annotations: [] }, + }), + ) + } queue.enqueue( sseEvent("response.output_item.done", { type: "response.output_item.done", diff --git a/index.test.js b/index.test.js index f07c01d..4016883 100644 --- a/index.test.js +++ b/index.test.js @@ -80,6 +80,21 @@ function createStreamingClient(chunks) { } } +function parseSseStream(text) { + // Parses SSE `event: \ndata: \n\n` chunks into an ordered array. + // Local to this test file; not exported. + return text + .split("\n\n") + .filter((block) => block.trim()) + .map((block) => { + const eventLine = block.match(/^event: (.+)$/m) + const dataLine = block.match(/^data: (.+)$/m) + if (!eventLine || !dataLine) return null + return { event: eventLine[1], data: JSON.parse(dataLine[1]) } + }) + .filter(Boolean) +} + test("OPTIONS preflight returns CORS headers", async () => { const handler = createProxyFetchHandler(createClient()) const request = new Request("http://127.0.0.1:4010/v1/models", { @@ -1108,6 +1123,68 @@ test("POST /v1/responses stream: true returns SSE lifecycle events", async () => assert.ok(text.includes("response.completed")) }) +test("POST /v1/responses stream: true emits content_part.done with accumulated text per OpenAI spec", async () => { + const events = [ + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: "The answer", + }, + }, + { + type: "message.part.updated", + properties: { + part: { sessionID: "sess-123", type: "text" }, + delta: " is 42.", + }, + }, + { type: "session.idle", properties: { sessionID: "sess-123" } }, + ] + + const handler = createProxyFetchHandler(createStreamingClient(events)) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + input: "What is 6 times 7?", + }), + }) + + const response = await handler(request) + const text = await response.text() + const parsed = parseSseStream(text) + const names = parsed.map((e) => e.event) + + // Discriminator 1 (gap #3): output_text.done.text must be the accumulated content + const outputTextDone = parsed.find((e) => e.event === "response.output_text.done") + assert.ok(outputTextDone, "response.output_text.done event must be present") + assert.equal(outputTextDone.data.text, "The answer is 42.") + + // Discriminator 2 (gap #2): content_part.done must be present with populated part.text + const contentPartDone = parsed.find((e) => e.event === "response.content_part.done") + assert.ok(contentPartDone, "response.content_part.done event must be present") + assert.equal(contentPartDone.data.part.type, "output_text") + assert.equal(contentPartDone.data.part.text, "The answer is 42.") + assert.deepEqual(contentPartDone.data.part.annotations, []) + + // Ordering: output_text.done -> content_part.done -> output_item.done + const idxOutputTextDone = names.indexOf("response.output_text.done") + const idxContentPartDone = names.indexOf("response.content_part.done") + const idxOutputItemDone = names.indexOf("response.output_item.done") + assert.ok(idxOutputTextDone >= 0, "output_text.done must be in the stream") + assert.ok( + idxContentPartDone > idxOutputTextDone, + "content_part.done must follow output_text.done", + ) + assert.ok( + idxOutputItemDone > idxContentPartDone, + "output_item.done must follow content_part.done", + ) +}) + test("POST /v1/responses stream: true with session.error emits response.failed", async () => { const events = [ { From d01cf203c4d307c995283b24be752680398d02c8 Mon Sep 17 00:00:00 2001 From: KochC Date: Sun, 5 Jul 2026 01:22:53 +0200 Subject: [PATCH 11/11] feat: add tool/function calling support (#52) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add tool/function calling support (closes #50) Adds OpenAI-style function tools, Anthropic tools, and Gemini function declarations across all four API surfaces (/v1/chat/completions, /v1/responses, /v1/messages, /v1beta/models/:model:generateContent), both streaming and non-streaming. OpenCode's own agent loop always executes tools itself, server-side, and has no concept of a 'client-executed' tool call to hand off to a caller. To bridge that gap: - When a request includes tools, the proxy dynamically registers a small local MCP server (mcp-tool-bridge.js) whose tool list is exactly the caller's declared tool schemas, reused from a small fixed-size pool of slot names (OpenCode's server API has no endpoint to deregister an MCP server once added). - Only those tools are enabled for that one prompt call via the existing tools enable/disable map; every built-in OpenCode tool stays disabled, same as before. - As soon as the model proposes calling one of the bridge tools, the full call (name + arguments) is already present on OpenCode's event stream (ToolStatePending includes the parsed input even before execution starts) - the proxy captures it and immediately aborts the session before the bridge's no-op tools/call handler would ever be consulted, then translates the call into the caller's expected tool_calls / tool_use / functionCall shape instead of a text answer. Also extends the OpenAI/Anthropic/Gemini/Responses message normalizers to render prior tool_calls/tool_use/functionCall and their results/tool_result/functionResponse as descriptive text when replaying conversation history, so multi-turn tool use works end-to-end even though sessions are stateless per-request. - index.js: parseOpenAITools/parseAnthropicTools/parseGeminiTools + applyOpenAIToolChoice/applyAnthropicToolChoice/applyGeminiToolChoice, sanitizeToolName, tool bridge pool + registerToolBridge, unified runAgentTurn (event-driven turn execution shared by executePrompt and executePromptStreaming when tools are present), tool-call branches in all four response builders and SSE emitters. - mcp-tool-bridge.js: minimal MCP stdio JSON-RPC server exposing caller-supplied tool schemas; tools/call is a harmless no-op since the proxy aborts the session before it would ever be consulted. - index.test.js: unit tests for the new parse/tool_choice helpers and history round-tripping, plus end-to-end tool-calling tests for all four API formats (stream + non-stream). - README.md: documents the new tools/tool_choice/toolConfig request fields, how the bridge mechanism works, its current limitations, and the new OPENCODE_LLM_PROXY_TOOL_BRIDGE_POOL_SIZE env var. Testing: - npm test — 138 passed (116 existing + 22 new) - npm run lint — clean * docs: feature tool calling prominently in README, expand discoverability keywords - Move the Tool calling section up (right after Configuration) and add a runnable curl request/response example, instead of burying it near the bottom after How it works. - Add a Contents section now that the README has grown to 10+ sections. - Call out tool calling in the top-level tagline, architecture diagram, supported-formats table, and Why section (coding agents are now a first-class use case, not just chat clients). - Note in Install that copying just index.js doesn't get you tool calling (needs mcp-tool-bridge.js alongside it) - use the npm plugin instead. - package.json: mention tool/function calling in the description and add tool-calling/function-calling/tools/mcp/model-context-protocol/ coding-agent/ai-agent/agentic keywords for npm search discoverability. --------- Co-authored-by: Framewrk CI --- README.md | 99 +++++- index.js | 765 ++++++++++++++++++++++++++++++++++++++------- index.test.js | 443 ++++++++++++++++++++++++++ mcp-tool-bridge.js | 107 +++++++ package.json | 12 +- 5 files changed, 1307 insertions(+), 119 deletions(-) create mode 100644 mcp-tool-bridge.js diff --git a/README.md b/README.md index 099ca1a..98acc45 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,12 @@ [![CI](https://github.com/KochC/opencode-llm-proxy/actions/workflows/ci.yml/badge.svg)](https://github.com/KochC/opencode-llm-proxy/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -**One local endpoint. Every model you have access to. Any API format.** +**One local endpoint. Every model you have access to. Any API format. Tool calling included.** opencode-llm-proxy is an [OpenCode](https://opencode.ai) plugin that starts a local HTTP server on `http://127.0.0.1:4010`. It translates between the API format your tool speaks and whichever LLM provider OpenCode has configured — so you never reconfigure the same models twice. ``` -Your tool (OpenAI / Anthropic / Gemini SDK) +Your tool (OpenAI / Anthropic / Gemini SDK, coding agent, etc.) │ ▼ http://127.0.0.1:4010 opencode-llm-proxy @@ -19,7 +19,7 @@ Your tool (OpenAI / Anthropic / Gemini SDK) GitHub Copilot · Anthropic · Gemini · Ollama · OpenRouter · Bedrock · … ``` -**Supported API formats — all with streaming:** +**Supported API formats — all with streaming and [tool/function calling](#tool-calling):** | Format | Endpoint | |---|---| @@ -28,6 +28,24 @@ Your tool (OpenAI / Anthropic / Gemini SDK) | Anthropic Messages API | `POST /v1/messages` | | Google Gemini | `POST /v1beta/models/:model:generateContent` | +**✨ Tool calling works with all four formats** — point a coding agent (Claude Code, Cursor, Continue, Cline, your own agent loop, ...) at the proxy and its `tools`/`tool_choice` calls are translated through to whatever model OpenCode has configured, with a real `tool_calls` / `tool_use` / `functionCall` response handed back. See [Tool calling](#tool-calling). + +--- + +## Contents + +- [Why](#why) +- [Quickstart](#quickstart) +- [Install](#install) +- [Configuration](#configuration) +- [Tool calling](#tool-calling) +- [Using with SDKs and tools](#using-with-sdks-and-tools) +- [Finding model IDs](#finding-model-ids) +- [API reference](#api-reference) +- [How it works](#how-it-works) +- [Limitations](#limitations) +- [License](#license) + --- ## Why @@ -41,6 +59,7 @@ Most LLM tools speak exactly one API dialect. OpenCode already manages connectio - You want to **swap models without code changes**. Your app talks to the proxy; you change the model in OpenCode config. - You want to **share your models on a LAN**. Expose the proxy on `0.0.0.0` and give teammates the URL. - You use the **Anthropic SDK** but want to route through GitHub Copilot or Bedrock. No code change in the SDK — just point it at the proxy. +- You're building or running a **coding agent** that needs real tool/function calling (read files, run shell commands, etc.) against whatever model OpenCode has configured. See [Tool calling](#tool-calling). --- @@ -110,6 +129,8 @@ curl -o .opencode/plugins/llm-proxy.js \ https://raw.githubusercontent.com/KochC/opencode-llm-proxy/main/index.js ``` +> Copying just `index.js` works for everything except [tool calling](#tool-calling), which also needs `mcp-tool-bridge.js` alongside it. Use the npm plugin install method if you want tool calling. + --- ## Configuration @@ -120,6 +141,7 @@ curl -o .opencode/plugins/llm-proxy.js \ | `OPENCODE_LLM_PROXY_PORT` | `4010` | TCP port. | | `OPENCODE_LLM_PROXY_TOKEN` | _(unset)_ | Bearer token required on every request. Unset = no auth. | | `OPENCODE_LLM_PROXY_CORS_ORIGIN` | `*` | `Access-Control-Allow-Origin` value for browser clients. | +| `OPENCODE_LLM_PROXY_TOOL_BRIDGE_POOL_SIZE` | `8` | Max concurrent in-flight requests using [tool calling](#tool-calling). | ```bash OPENCODE_LLM_PROXY_HOST=0.0.0.0 \ @@ -129,6 +151,67 @@ opencode --- +## Tool calling + +The proxy supports real tool/function calling on **all four API formats** — OpenAI function tools (`tools` on `/v1/chat/completions` and `/v1/responses`), Anthropic tools (`tools` on `/v1/messages`), and Gemini function declarations (`tools` on `:generateContent`/`:streamGenerateContent`). This is what lets coding agents and other tool-using clients work through the proxy, not just plain chat. + +```bash +curl http://127.0.0.1:4010/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "github-copilot/claude-sonnet-4.6", + "messages": [{"role": "user", "content": "What is the weather in NYC?"}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { "city": { "type": "string" } }, + "required": ["city"] + } + } + }] + }' +``` + +```json +{ + "choices": [{ + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "call_...", + "type": "function", + "function": { "name": "get_weather", "arguments": "{\"city\":\"NYC\"}" } + }] + } + }] +} +``` + +Send the tool's result back on your next request (`role: "tool"` / `tool_result` / `functionResponse`, per your API's convention) alongside the full conversation history, same as any other multi-turn request — the proxy is stateless between calls either way. + +### How tool calling works under the hood + +OpenCode's own agent loop always executes tools itself, server-side, so there's no native concept of a "client-executed" tool call to hand off to. To bridge that gap, when a request includes `tools`: + +1. The proxy dynamically registers a small local [MCP](https://opencode.ai/docs/mcp-servers/) server whose tool list is exactly your declared tool schemas (see `mcp-tool-bridge.js`). +2. Only those tools are enabled for that one prompt call — every built-in OpenCode tool stays disabled, same as always. +3. As soon as the model proposes calling one of your tools, the proxy immediately aborts the OpenCode session (before the bridge's no-op handler is ever consulted) and translates the captured call name + arguments into your API's tool-call shape — `tool_calls` (OpenAI), `tool_use` (Anthropic), or a `functionCall` part (Gemini) — instead of a text answer. + +### Notes and current limitations + +- One tool call per turn — parallel/multiple simultaneous tool calls aren't supported. +- `tool_choice: "none"` (OpenAI/Gemini `mode: "NONE"`/Anthropic `type: "none"`) disables tool calling for that request; forcing a specific named tool is supported. +- Bridge servers are reused from a small fixed-size pool (`px_tools_0`, `px_tools_1`, ...) rather than registered fresh per request, since OpenCode's server API has no endpoint to deregister an MCP server once added. Configure the pool size with `OPENCODE_LLM_PROXY_TOOL_BRIDGE_POOL_SIZE` (default `8`) if you expect more than 8 concurrent in-flight tool-calling requests. +- The bridge process is spawned with `node`, so `node` must be on `PATH` wherever OpenCode is running. + +--- + ## Using with SDKs and tools ### OpenAI SDK (JS/TS) @@ -310,18 +393,18 @@ x-opencode-provider: anthropic Returns all models from all configured providers in OpenAI list format. ### POST /v1/chat/completions -OpenAI Chat Completions. Required fields: `model`, `messages`. Optional: `stream`, `temperature`, `max_tokens`. +OpenAI Chat Completions. Required fields: `model`, `messages`. Optional: `stream`, `temperature`, `max_tokens`, `tools`, `tool_choice`. ### POST /v1/responses -OpenAI Responses API. Required fields: `model`, `input`. Optional: `instructions`, `stream`, `max_output_tokens`. +OpenAI Responses API. Required fields: `model`, `input`. Optional: `instructions`, `stream`, `max_output_tokens`, `tools`, `tool_choice`. ### POST /v1/messages -Anthropic Messages API. Required fields: `model`, `messages`. Optional: `system` (string or array of `{type: "text", text: string}` content blocks), `max_tokens`, `stream`. +Anthropic Messages API. Required fields: `model`, `messages`. Optional: `system` (string or array of `{type: "text", text: string}` content blocks), `max_tokens`, `stream`, `tools`, `tool_choice`. Errors are returned in Anthropic format: `{ "type": "error", "error": { "type": "...", "message": "..." } }`. ### POST /v1beta/models/:model:generateContent -Google Gemini non-streaming. Model name in URL path. Required field: `contents`. Optional: `systemInstruction`, `generationConfig`. +Google Gemini non-streaming. Model name in URL path. Required field: `contents`. Optional: `systemInstruction`, `generationConfig`, `tools`, `toolConfig`. ### POST /v1beta/models/:model:streamGenerateContent Same as above, returns newline-delimited JSON stream. @@ -345,9 +428,9 @@ Streaming uses OpenCode's `client.event.subscribe()` SSE stream. Text deltas are ## Limitations - Text only — image, audio, and file inputs are ignored -- No tool/function calling — all OpenCode tools are disabled for proxy sessions - No cross-request session state — send full conversation history on every request - Temperature and max tokens are advisory (passed as system prompt hints) +- Tool calling supports one call per turn — see [Tool calling](#tool-calling) above --- diff --git a/index.js b/index.js index a07678e..e22e5e6 100644 --- a/index.js +++ b/index.js @@ -1,4 +1,7 @@ +import { fileURLToPath } from "node:url" + const STATE_KEY = "__opencodeOpenAIProxyState" +const BRIDGE_SCRIPT_PATH = fileURLToPath(new URL("./mcp-tool-bridge.js", import.meta.url)) function getState() { if (!globalThis[STATE_KEY]) { @@ -114,11 +117,34 @@ export function toTextContent(content) { } export function normalizeMessages(messages) { + const toolNameByCallId = new Map() + return messages - .map((message) => ({ - role: message.role, - content: toTextContent(message.content).trim(), - })) + .map((message) => { + if (message.role === "assistant" && Array.isArray(message.tool_calls) && message.tool_calls.length > 0) { + const baseText = toTextContent(message.content).trim() + const callsText = message.tool_calls + .map((call) => { + const name = call.function?.name ?? call.name ?? "unknown_tool" + const args = call.function?.arguments ?? "" + if (call.id) toolNameByCallId.set(call.id, name) + return `[Called tool ${name} with arguments ${args}]` + }) + .join("\n") + return { role: message.role, content: [baseText, callsText].filter(Boolean).join("\n\n") } + } + + if (message.role === "tool") { + const name = toolNameByCallId.get(message.tool_call_id) ?? "tool" + const resultText = toTextContent(message.content).trim() + return { role: "tool", content: `[Result from tool ${name}]: ${resultText}` } + } + + return { + role: message.role, + content: toTextContent(message.content).trim(), + } + }) .filter((message) => message.content.length > 0) } @@ -129,8 +155,22 @@ export function normalizeResponseInput(input) { if (!Array.isArray(input)) return [] + const toolNameByCallId = new Map() + return input .map((item) => { + if (item?.type === "function_call") { + const name = item.name ?? "unknown_tool" + if (item.call_id) toolNameByCallId.set(item.call_id, name) + return { role: "assistant", content: `[Called tool ${name} with arguments ${item.arguments ?? ""}]` } + } + + if (item?.type === "function_call_output") { + const name = toolNameByCallId.get(item.call_id) ?? "tool" + const output = typeof item.output === "string" ? item.output : JSON.stringify(item.output ?? "") + return { role: "tool", content: `[Result from tool ${name}]: ${output}` } + } + const role = item.role ?? item.type ?? "user" if (typeof item.content === "string") { return { role, content: item.content.trim() } @@ -227,7 +267,28 @@ export function extractAssistantText(parts) { .trim() } -async function executePrompt(client, request, model, messages, system) { +async function executePrompt(client, request, model, messages, system, callerTools = []) { + if (Array.isArray(callerTools) && callerTools.length > 0) { + // Tool-aware path: must watch the event stream (via runAgentTurn) rather than + // block on session.prompt, so we can intercept a proposed tool call instead of + // letting OpenCode's agent loop run to a final text answer. + const result = await runAgentTurn(client, model, messages, system, callerTools, () => {}) + return { + content: result.content, + toolCall: result.toolCall, + request, + sessionID: result.sessionID, + completion: { + data: { + info: { + finish: result.finish, + tokens: result.tokens, + }, + }, + }, + } + } + const tools = await getDisabledTools(client) const session = await client.session.create({ body: { @@ -263,77 +324,48 @@ async function executePrompt(client, request, model, messages, system) { return { content, + toolCall: null, completion, request, sessionID: session.data.id, } } -async function executePromptStreaming(client, model, messages, system, onChunk) { - const tools = await getDisabledTools(client) - const session = await client.session.create({ - body: { title: `Proxy: ${model.id}` }, - }) - const sessionID = session.data.id - const prompt = buildPrompt(messages) - - // Subscribe to the event stream before sending the prompt so we don't miss events. - const { stream } = await client.event.subscribe() - - await client.session.promptAsync({ - path: { id: sessionID }, - body: { - model: { providerID: model.providerID, modelID: model.modelID }, - system, - tools, - parts: [{ type: "text", text: prompt }], - }, - }) - - let errorMessage = null - - for await (const event of stream) { - if (event.type === "message.part.updated") { - const part = event.properties?.part - const delta = event.properties?.delta - if ( - part?.sessionID === sessionID && - part?.type === "text" && - typeof delta === "string" && - delta.length > 0 - ) { - onChunk(delta) - } - } else if (event.type === "session.error") { - if (!event.properties?.sessionID || event.properties.sessionID === sessionID) { - errorMessage = event.properties?.error?.message ?? "Model call failed." - } - } else if (event.type === "session.idle") { - if (event.properties?.sessionID === sessionID) { - break - } - } - } - - if (errorMessage) { - throw new Error(errorMessage) - } - - // Fetch final message to get token usage. - const messages_ = await client.session.messages({ path: { id: sessionID } }) - const assistantMsg = (messages_.data ?? []) - .filter((m) => m.role === "assistant") - .at(-1) - +async function executePromptStreaming(client, model, messages, system, onChunk, callerTools = []) { + const result = await runAgentTurn(client, model, messages, system, callerTools, onChunk) return { - sessionID, - tokens: assistantMsg?.tokens ?? { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, - finish: assistantMsg?.finish, + sessionID: result.sessionID, + tokens: result.tokens, + finish: result.finish, + toolCall: result.toolCall, } } function createChatCompletionResponse(result, model) { const now = Math.floor(Date.now() / 1000) + const tokensIn = result.completion.data.info?.tokens?.input ?? 0 + const tokensOut = result.completion.data.info?.tokens?.output ?? 0 + + const message = result.toolCall + ? { + role: "assistant", + content: null, + tool_calls: [ + { + id: result.toolCall.id, + type: "function", + function: { + name: result.toolCall.name, + arguments: JSON.stringify(result.toolCall.arguments ?? {}), + }, + }, + ], + } + : { + role: "assistant", + content: result.content, + } + return { id: `chatcmpl_${crypto.randomUUID().replace(/-/g, "")}`, object: "chat.completion", @@ -342,19 +374,14 @@ function createChatCompletionResponse(result, model) { choices: [ { index: 0, - finish_reason: mapFinishReason(result.completion.data.info?.finish), - message: { - role: "assistant", - content: result.content, - }, + finish_reason: result.toolCall ? "tool_calls" : mapFinishReason(result.completion.data.info?.finish), + message, }, ], usage: { - prompt_tokens: result.completion.data.info?.tokens?.input ?? 0, - completion_tokens: result.completion.data.info?.tokens?.output ?? 0, - total_tokens: - (result.completion.data.info?.tokens?.input ?? 0) + - (result.completion.data.info?.tokens?.output ?? 0), + prompt_tokens: tokensIn, + completion_tokens: tokensOut, + total_tokens: tokensIn + tokensOut, }, } } @@ -363,28 +390,41 @@ function createResponsesApiResponse(result, model) { const tokensIn = result.completion.data.info?.tokens?.input ?? 0 const tokensOut = result.completion.data.info?.tokens?.output ?? 0 + const output = result.toolCall + ? [ + { + id: `fc_${crypto.randomUUID().replace(/-/g, "")}`, + type: "function_call", + call_id: result.toolCall.id, + name: result.toolCall.name, + arguments: JSON.stringify(result.toolCall.arguments ?? {}), + status: "completed", + }, + ] + : [ + { + id: `msg_${crypto.randomUUID().replace(/-/g, "")}`, + type: "message", + status: "completed", + role: "assistant", + content: [ + { + type: "output_text", + text: result.content, + annotations: [], + }, + ], + }, + ] + return { id: `resp_${crypto.randomUUID().replace(/-/g, "")}`, object: "response", created_at: Math.floor(Date.now() / 1000), status: "completed", model: model.id, - output: [ - { - id: `msg_${crypto.randomUUID().replace(/-/g, "")}`, - type: "message", - status: "completed", - role: "assistant", - content: [ - { - type: "output_text", - text: result.content, - annotations: [], - }, - ], - }, - ], - output_text: result.content, + output, + output_text: result.toolCall ? "" : result.content, parallel_tool_calls: false, reasoning: { effort: result.request.reasoning?.effort ?? null, @@ -440,6 +480,314 @@ async function getDisabledTools(client) { return state.toolOffSwitch } +// --------------------------------------------------------------------------- +// Tool calling support +// +// OpenCode's own agent loop always executes tools itself, server-side, and has +// no concept of a "client-executed" tool call. To offer OpenAI/Anthropic/Gemini +// style tool calling (propose a call, hand control back to the caller, resume +// once they supply a result) we: +// +// 1. Dynamically register a tiny local MCP server ("bridge") whose tool list +// is exactly the caller's declared tool schemas (see mcp-tool-bridge.js). +// 2. Enable only those tool IDs for this one prompt call. +// 3. Watch OpenCode's event stream. As soon as the model proposes calling one +// of the bridge tools, the full call (name + arguments) is already present +// on the event (see ToolStatePending in OpenCode's SDK types) - we grab it +// and immediately abort the session before the bridge's harmless no-op +// tools/call handler would ever matter. +// 4. Translate the captured call into the caller's expected tool-call shape. +// +// Bridge servers are reused from a small fixed-size pool of slot names (rather +// than registered fresh per request) since OpenCode's server API exposes no way +// to remove/deregister an MCP server once added. +// --------------------------------------------------------------------------- + +function getToolBridgeState() { + const state = getState() + if (!state.toolBridge) { + const configured = Number.parseInt(process.env.OPENCODE_LLM_PROXY_TOOL_BRIDGE_POOL_SIZE ?? "", 10) + const poolSize = Number.isFinite(configured) && configured > 0 ? configured : 8 + state.toolBridge = { + freeSlots: Array.from({ length: poolSize }, (_, i) => `px_tools_${i}`), + waiters: [], + } + } + return state.toolBridge +} + +async function acquireBridgeSlot() { + const bridgeState = getToolBridgeState() + if (bridgeState.freeSlots.length > 0) { + return bridgeState.freeSlots.shift() + } + return new Promise((resolve) => { + bridgeState.waiters.push(resolve) + }) +} + +function releaseBridgeSlot(slotName) { + const bridgeState = getToolBridgeState() + if (bridgeState.waiters.length > 0) { + const resolve = bridgeState.waiters.shift() + resolve(slotName) + } else { + bridgeState.freeSlots.push(slotName) + } +} + +export function sanitizeToolName(name, seen = new Set()) { + let sanitized = String(name ?? "") + .replace(/[^a-zA-Z0-9_]/g, "_") + .slice(0, 60) + if (!sanitized) sanitized = "tool" + if (!/^[a-zA-Z_]/.test(sanitized)) sanitized = `t_${sanitized}` + + let candidate = sanitized + let suffix = 2 + while (seen.has(candidate)) { + candidate = `${sanitized}_${suffix}` + suffix++ + } + seen.add(candidate) + return candidate +} + +function normalizeParameters(parameters) { + if (parameters && typeof parameters === "object") return parameters + return { type: "object", properties: {} } +} + +export function parseOpenAITools(body) { + const list = [] + if (Array.isArray(body?.tools)) { + for (const entry of body.tools) { + if (!entry || entry.type !== "function") continue + // Chat Completions nests fields under `function`; the Responses API uses a flat shape. + const fn = entry.function ?? entry + if (typeof fn.name === "string" && fn.name) { + list.push({ + name: fn.name, + description: typeof fn.description === "string" ? fn.description : "", + parameters: normalizeParameters(fn.parameters), + }) + } + } + } else if (Array.isArray(body?.functions)) { + // Legacy (pre-2023-08) OpenAI `functions` field. + for (const fn of body.functions) { + if (fn && typeof fn.name === "string" && fn.name) { + list.push({ + name: fn.name, + description: typeof fn.description === "string" ? fn.description : "", + parameters: normalizeParameters(fn.parameters), + }) + } + } + } + return list +} + +export function applyOpenAIToolChoice(tools, toolChoice) { + if (toolChoice === "none") return [] + if (toolChoice && typeof toolChoice === "object") { + const name = toolChoice.function?.name ?? toolChoice.name + if (toolChoice.type === "function" && name) { + return tools.filter((tool) => tool.name === name) + } + } + return tools +} + +export function parseAnthropicTools(body) { + const list = [] + if (Array.isArray(body?.tools)) { + for (const tool of body.tools) { + if (tool && typeof tool.name === "string" && tool.name) { + list.push({ + name: tool.name, + description: typeof tool.description === "string" ? tool.description : "", + parameters: normalizeParameters(tool.input_schema), + }) + } + } + } + return list +} + +export function applyAnthropicToolChoice(tools, toolChoice) { + if (toolChoice?.type === "none") return [] + if (toolChoice?.type === "tool" && toolChoice.name) { + return tools.filter((tool) => tool.name === toolChoice.name) + } + return tools +} + +export function parseGeminiTools(body) { + const list = [] + if (Array.isArray(body?.tools)) { + for (const toolGroup of body.tools) { + const declarations = Array.isArray(toolGroup?.functionDeclarations) ? toolGroup.functionDeclarations : [] + for (const decl of declarations) { + if (decl && typeof decl.name === "string" && decl.name) { + list.push({ + name: decl.name, + description: typeof decl.description === "string" ? decl.description : "", + parameters: normalizeParameters(decl.parameters), + }) + } + } + } + } + return list +} + +export function applyGeminiToolChoice(tools, toolConfig) { + const mode = toolConfig?.functionCallingConfig?.mode + if (mode === "NONE") return [] + const allowed = toolConfig?.functionCallingConfig?.allowedFunctionNames + if (Array.isArray(allowed) && allowed.length > 0) { + return tools.filter((tool) => allowed.includes(tool.name)) + } + return tools +} + +async function registerToolBridge(client, tools) { + const slotName = await acquireBridgeSlot() + const seen = new Set() + const nameMap = new Map() // full bridge tool ID ("_") -> original caller-facing name + const bridgeTools = tools.map((tool) => { + const sanitized = sanitizeToolName(tool.name, seen) + nameMap.set(`${slotName}_${sanitized}`, tool.name) + return { name: sanitized, description: tool.description, parameters: tool.parameters } + }) + + try { + // Force a fresh respawn so the bridge process picks up this request's tool schema. + await client.mcp.disconnect({ path: { name: slotName } }) + } catch { + // Not previously connected; nothing to do. + } + + await client.mcp.add({ + body: { + name: slotName, + config: { + type: "local", + command: ["node", BRIDGE_SCRIPT_PATH], + environment: { + OPENCODE_LLM_PROXY_BRIDGE_TOOLS: JSON.stringify(bridgeTools), + }, + timeout: 10000, + }, + }, + }) + + const toolIDs = bridgeTools.map((tool) => `${slotName}_${tool.name}`) + return { slotName, toolIDs, nameMap } +} + +function releaseToolBridge(bridge) { + if (bridge) releaseBridgeSlot(bridge.slotName) +} + +async function runAgentTurn(client, model, messages, system, callerTools, onChunk) { + const baseTools = await getDisabledTools(client) + let toolsMap = baseTools + let bridge = null + + if (Array.isArray(callerTools) && callerTools.length > 0) { + bridge = await registerToolBridge(client, callerTools) + toolsMap = { ...baseTools } + for (const id of bridge.toolIDs) toolsMap[id] = true + } + + const session = await client.session.create({ body: { title: `Proxy: ${model.id}` } }) + const sessionID = session.data.id + const prompt = buildPrompt(messages) + const toolIDSet = bridge ? new Set(bridge.toolIDs) : null + + // Subscribe to the event stream before sending the prompt so we don't miss events. + const { stream } = await client.event.subscribe() + + await client.session.promptAsync({ + path: { id: sessionID }, + body: { + model: { providerID: model.providerID, modelID: model.modelID }, + system, + tools: toolsMap, + parts: [{ type: "text", text: prompt }], + }, + }) + + let errorMessage = null + let content = "" + let toolCall = null + + try { + for await (const event of stream) { + if (event.type === "message.part.updated") { + const part = event.properties?.part + const delta = event.properties?.delta + + if ( + part?.sessionID === sessionID && + part?.type === "text" && + typeof delta === "string" && + delta.length > 0 + ) { + content += delta + onChunk?.(delta) + } else if ( + toolIDSet && + part?.sessionID === sessionID && + part?.type === "tool" && + toolIDSet.has(part.tool) && + (part.state?.status === "pending" || part.state?.status === "running") + ) { + toolCall = { + id: part.callID, + name: bridge.nameMap.get(part.tool) ?? part.tool, + arguments: part.state.input ?? {}, + } + try { + await client.session.abort({ path: { id: sessionID } }) + } catch { + // Best effort - we're ending our own read loop regardless. + } + break + } + } else if (event.type === "session.error") { + if (!event.properties?.sessionID || event.properties.sessionID === sessionID) { + errorMessage = event.properties?.error?.message ?? "Model call failed." + } + break + } else if (event.type === "session.idle") { + if (event.properties?.sessionID === sessionID) { + break + } + } + } + } finally { + releaseToolBridge(bridge) + } + + if (errorMessage && !toolCall) { + throw new Error(errorMessage) + } + + const messagesResult = await client.session.messages({ path: { id: sessionID } }) + const assistantMsg = (messagesResult.data ?? []).filter((m) => m.role === "assistant").at(-1) + + return { + sessionID, + content, + toolCall, + tokens: assistantMsg?.tokens ?? { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + finish: toolCall ? "tool_calls" : assistantMsg?.finish, + } +} + async function listModels(client) { const result = await client.config.providers() const payload = result.data @@ -575,6 +923,8 @@ function createModelResponse(models) { // --------------------------------------------------------------------------- export function normalizeAnthropicMessages(messages) { + const toolNameByUseId = new Map() + return messages .map((message) => { let content = "" @@ -582,8 +932,30 @@ export function normalizeAnthropicMessages(messages) { content = message.content.trim() } else if (Array.isArray(message.content)) { content = message.content - .filter((block) => block && block.type === "text" && typeof block.text === "string") - .map((block) => block.text.trim()) + .map((block) => { + if (!block) return "" + if (block.type === "text" && typeof block.text === "string") { + return block.text.trim() + } + if (block.type === "tool_use") { + if (block.id) toolNameByUseId.set(block.id, block.name) + return `[Called tool ${block.name} with arguments ${JSON.stringify(block.input ?? {})}]` + } + if (block.type === "tool_result") { + const name = toolNameByUseId.get(block.tool_use_id) ?? "tool" + let resultText = "" + if (typeof block.content === "string") { + resultText = block.content + } else if (Array.isArray(block.content)) { + resultText = block.content + .filter((inner) => inner && inner.type === "text" && typeof inner.text === "string") + .map((inner) => inner.text) + .join("\n\n") + } + return `[Result from tool ${name}]: ${resultText}` + } + return "" + }) .filter(Boolean) .join("\n\n") } @@ -618,13 +990,24 @@ export function mapFinishReasonToAnthropic(finish) { function createAnthropicResponse(result, model) { const tokensIn = result.completion.data.info?.tokens?.input ?? 0 const tokensOut = result.completion.data.info?.tokens?.output ?? 0 + const content = result.toolCall + ? [ + { + type: "tool_use", + id: result.toolCall.id, + name: result.toolCall.name, + input: result.toolCall.arguments ?? {}, + }, + ] + : [{ type: "text", text: result.content }] + return { id: `msg_${crypto.randomUUID().replace(/-/g, "")}`, type: "message", role: "assistant", - content: [{ type: "text", text: result.content }], + content, model: model.id, - stop_reason: mapFinishReasonToAnthropic(result.completion.data.info?.finish), + stop_reason: result.toolCall ? "tool_use" : mapFinishReasonToAnthropic(result.completion.data.info?.finish), stop_sequence: null, usage: { input_tokens: tokensIn, output_tokens: tokensOut }, } @@ -659,7 +1042,17 @@ export function normalizeGeminiContents(contents) { const role = item.role === "model" ? "assistant" : (item.role ?? "user") const content = Array.isArray(item.parts) ? item.parts - .map((part) => (typeof part?.text === "string" ? part.text.trim() : "")) + .map((part) => { + if (!part) return "" + if (typeof part.text === "string") return part.text.trim() + if (part.functionCall) { + return `[Called tool ${part.functionCall.name} with arguments ${JSON.stringify(part.functionCall.args ?? {})}]` + } + if (part.functionResponse) { + return `[Result from tool ${part.functionResponse.name}]: ${JSON.stringify(part.functionResponse.response ?? {})}` + } + return "" + }) .filter(Boolean) .join("\n\n") : "" @@ -687,11 +1080,15 @@ export function mapFinishReasonToGemini(finish) { return "STOP" } -function createGeminiResponse(content, finish, tokens) { +function createGeminiResponse(content, finish, tokens, toolCall) { + const parts = toolCall + ? [{ functionCall: { name: toolCall.name, args: toolCall.arguments ?? {} } }] + : [{ text: content }] + return { candidates: [ { - content: { role: "model", parts: [{ text: content }] }, + content: { role: "model", parts }, finishReason: mapFinishReasonToGemini(finish), index: 0, }, @@ -773,6 +1170,7 @@ export function createProxyFetchHandler(client) { } const system = buildSystemPrompt(messages, body) + const callerTools = applyOpenAIToolChoice(parseOpenAITools(body), body.tool_choice) if (body.stream) { const completionID = `chatcmpl_${crypto.randomUUID().replace(/-/g, "")}` @@ -796,14 +1194,51 @@ export function createProxyFetchHandler(client) { }) queue.enqueue(`data: ${chunk}\n\n`) }, + callerTools, ) .then((streamResult) => { + if (streamResult.toolCall) { + const toolCallChunk = JSON.stringify({ + id: completionID, + object: "chat.completion.chunk", + created: now, + model: model.id, + choices: [ + { + index: 0, + delta: { + role: "assistant", + tool_calls: [ + { + index: 0, + id: streamResult.toolCall.id, + type: "function", + function: { + name: streamResult.toolCall.name, + arguments: JSON.stringify(streamResult.toolCall.arguments ?? {}), + }, + }, + ], + }, + finish_reason: null, + }, + ], + }) + queue.enqueue(`data: ${toolCallChunk}\n\n`) + } + const finalChunk = JSON.stringify({ id: completionID, object: "chat.completion.chunk", created: now, model: model.id, - choices: [{ index: 0, delta: {}, finish_reason: mapFinishReason(streamResult.finish) }], + choices: [ + { + index: 0, + delta: {}, + finish_reason: streamResult.toolCall ? "tool_calls" : mapFinishReason(streamResult.finish), + }, + ], usage: { prompt_tokens: streamResult.tokens.input, completion_tokens: streamResult.tokens.output, @@ -836,7 +1271,7 @@ export function createProxyFetchHandler(client) { } try { - const result = await executePrompt(client, body, model, messages, system) + const result = await executePrompt(client, body, model, messages, system, callerTools) return json(createChatCompletionResponse(result, model), 200, {}, request) } catch (error) { const message = error instanceof Error ? error.message : String(error) @@ -875,6 +1310,7 @@ export function createProxyFetchHandler(client) { max_tokens: body.max_output_tokens, max_completion_tokens: body.max_output_tokens, }) + const callerTools = applyOpenAIToolChoice(parseOpenAITools(body), body.tool_choice) let model try { @@ -955,8 +1391,76 @@ export function createProxyFetchHandler(client) { }), ) }, + callerTools, ) .then((streamResult) => { + if (streamResult.toolCall) { + const args = JSON.stringify(streamResult.toolCall.arguments ?? {}) + const callItemID = `fc_${crypto.randomUUID().replace(/-/g, "")}` + queue.enqueue( + sseEvent("response.output_item.added", { + type: "response.output_item.added", + output_index: 0, + item: { + id: callItemID, + type: "function_call", + status: "in_progress", + call_id: streamResult.toolCall.id, + name: streamResult.toolCall.name, + arguments: "", + }, + }), + ) + queue.enqueue( + sseEvent("response.function_call_arguments.delta", { + type: "response.function_call_arguments.delta", + item_id: callItemID, + output_index: 0, + delta: args, + }), + ) + queue.enqueue( + sseEvent("response.function_call_arguments.done", { + type: "response.function_call_arguments.done", + item_id: callItemID, + output_index: 0, + arguments: args, + }), + ) + queue.enqueue( + sseEvent("response.output_item.done", { + type: "response.output_item.done", + output_index: 0, + item: { + id: callItemID, + type: "function_call", + status: "completed", + call_id: streamResult.toolCall.id, + name: streamResult.toolCall.name, + arguments: args, + }, + }), + ) + queue.enqueue( + sseEvent("response.completed", { + type: "response.completed", + response: { + id: responseID, + object: "response", + created_at: now, + status: "completed", + model: model.id, + usage: { + input_tokens: streamResult.tokens.input, + output_tokens: streamResult.tokens.output, + total_tokens: streamResult.tokens.input + streamResult.tokens.output, + }, + }, + }), + ) + return + } + queue.enqueue( sseEvent("response.output_text.done", { type: "response.output_text.done", @@ -1036,7 +1540,7 @@ export function createProxyFetchHandler(client) { } try { - const result = await executePrompt(client, body, model, messages, system) + const result = await executePrompt(client, body, model, messages, system, callerTools) return json(createResponsesApiResponse(result, model), 200, {}, request) } catch (error) { const message = error instanceof Error ? error.message : String(error) @@ -1085,6 +1589,7 @@ export function createProxyFetchHandler(client) { temperature: body.temperature, max_tokens: body.max_tokens, }) + const callerTools = applyAnthropicToolChoice(parseAnthropicTools(body), body.tool_choice) let model try { @@ -1118,26 +1623,64 @@ export function createProxyFetchHandler(client) { usage: { input_tokens: 0, output_tokens: 0 }, }, })) - queue.enqueue(sseEvent("content_block_start", { - type: "content_block_start", - index: 0, - content_block: { type: "text", text: "" }, - })) + let textBlockStarted = false const runPromise = executePromptStreaming( client, model, messages, system, (delta) => { + if (!textBlockStarted) { + queue.enqueue(sseEvent("content_block_start", { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + })) + textBlockStarted = true + } queue.enqueue(sseEvent("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: delta }, })) }, + callerTools, ) .then((streamResult) => { + if (streamResult.toolCall) { + if (textBlockStarted) { + queue.enqueue(sseEvent("content_block_stop", { type: "content_block_stop", index: 0 })) + } + const blockIndex = textBlockStarted ? 1 : 0 + const argsJson = JSON.stringify(streamResult.toolCall.arguments ?? {}) + queue.enqueue(sseEvent("content_block_start", { + type: "content_block_start", + index: blockIndex, + content_block: { type: "tool_use", id: streamResult.toolCall.id, name: streamResult.toolCall.name, input: {} }, + })) + queue.enqueue(sseEvent("content_block_delta", { + type: "content_block_delta", + index: blockIndex, + delta: { type: "input_json_delta", partial_json: argsJson }, + })) + queue.enqueue(sseEvent("content_block_stop", { type: "content_block_stop", index: blockIndex })) + queue.enqueue(sseEvent("message_delta", { + type: "message_delta", + delta: { stop_reason: "tool_use", stop_sequence: null }, + usage: { output_tokens: streamResult.tokens.output }, + })) + queue.enqueue(sseEvent("message_stop", { type: "message_stop" })) + return + } + + if (!textBlockStarted) { + queue.enqueue(sseEvent("content_block_start", { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + })) + } queue.enqueue(sseEvent("content_block_stop", { type: "content_block_stop", index: 0 })) queue.enqueue(sseEvent("message_delta", { type: "message_delta", @@ -1166,7 +1709,7 @@ export function createProxyFetchHandler(client) { } try { - const result = await executePrompt(client, body, model, messages, system) + const result = await executePrompt(client, body, model, messages, system, callerTools) return json(createAnthropicResponse(result, model), 200, {}, request) } catch (error) { const message = error instanceof Error ? error.message : String(error) @@ -1211,6 +1754,7 @@ export function createProxyFetchHandler(client) { temperature: body.generationConfig?.temperature, max_tokens: body.generationConfig?.maxOutputTokens, }) + const callerTools = applyGeminiToolChoice(parseGeminiTools(body), body.toolConfig) let model try { @@ -1235,10 +1779,13 @@ export function createProxyFetchHandler(client) { const chunk = JSON.stringify(createGeminiResponse(delta, null, null)) queue.enqueue(chunk + "\n") }, + callerTools, ) .then((streamResult) => { const finalChunk = JSON.stringify( - createGeminiResponse("", streamResult.finish, streamResult.tokens), + streamResult.toolCall + ? createGeminiResponse("", streamResult.finish, streamResult.tokens, streamResult.toolCall) + : createGeminiResponse("", streamResult.finish, streamResult.tokens), ) queue.enqueue(finalChunk + "\n") }) @@ -1283,10 +1830,10 @@ export function createProxyFetchHandler(client) { } try { - const result = await executePrompt(client, body, model, messages, system) + const result = await executePrompt(client, body, model, messages, system, callerTools) const finish = result.completion.data.info?.finish const tokens = result.completion.data.info?.tokens - return json(createGeminiResponse(result.content, finish, tokens), 200, {}, request) + return json(createGeminiResponse(result.content, finish, tokens, result.toolCall), 200, {}, request) } catch (error) { const message = error instanceof Error ? error.message : String(error) await safeLog(client, "error", "Gemini proxy call failed", { error: message, requestedModel: geminiModelName }) diff --git a/index.test.js b/index.test.js index 4016883..1958758 100644 --- a/index.test.js +++ b/index.test.js @@ -18,6 +18,13 @@ import { normalizeGeminiContents, extractGeminiSystemInstruction, mapFinishReasonToGemini, + sanitizeToolName, + parseOpenAITools, + applyOpenAIToolChoice, + parseAnthropicTools, + applyAnthropicToolChoice, + parseGeminiTools, + applyGeminiToolChoice, } from "./index.js" // --------------------------------------------------------------------------- @@ -1872,3 +1879,439 @@ test("POST /v1beta/models/:model:streamGenerateContent returns NDJSON stream", a assert.ok(text.includes("Gem")) assert.ok(text.includes("ini")) }) + +// --------------------------------------------------------------------------- +// Unit: tool parsing / tool_choice helpers +// --------------------------------------------------------------------------- + +test("sanitizeToolName replaces invalid characters and de-duplicates", () => { + assert.equal(sanitizeToolName("get_weather"), "get_weather") + assert.equal(sanitizeToolName("get-weather.v2"), "get_weather_v2") + assert.equal(sanitizeToolName("123start"), "t_123start") + assert.equal(sanitizeToolName(""), "tool") + + const seen = new Set() + assert.equal(sanitizeToolName("dup", seen), "dup") + assert.equal(sanitizeToolName("dup", seen), "dup_2") + assert.equal(sanitizeToolName("dup", seen), "dup_3") +}) + +test("parseOpenAITools extracts function tools (Chat Completions nested shape)", () => { + const tools = parseOpenAITools({ + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get the weather", + parameters: { type: "object", properties: { city: { type: "string" } } }, + }, + }, + { type: "function", function: { name: "no_params" } }, + { type: "not_function", function: { name: "ignored" } }, + ], + }) + + assert.equal(tools.length, 2) + assert.equal(tools[0].name, "get_weather") + assert.equal(tools[0].description, "Get the weather") + assert.deepEqual(tools[0].parameters, { type: "object", properties: { city: { type: "string" } } }) + assert.equal(tools[1].name, "no_params") + assert.deepEqual(tools[1].parameters, { type: "object", properties: {} }) +}) + +test("parseOpenAITools extracts function tools (Responses API flat shape)", () => { + const tools = parseOpenAITools({ + tools: [ + { type: "function", name: "get_weather", description: "Get weather", parameters: { type: "object" } }, + ], + }) + + assert.equal(tools.length, 1) + assert.equal(tools[0].name, "get_weather") +}) + +test("parseOpenAITools supports legacy 'functions' field", () => { + const tools = parseOpenAITools({ functions: [{ name: "legacy_fn", description: "d" }] }) + assert.equal(tools.length, 1) + assert.equal(tools[0].name, "legacy_fn") +}) + +test("parseOpenAITools returns empty array when no tools present", () => { + assert.deepEqual(parseOpenAITools({}), []) +}) + +test("applyOpenAIToolChoice filters to a single named function, or none, or unchanged", () => { + const tools = [{ name: "a", description: "", parameters: {} }, { name: "b", description: "", parameters: {} }] + assert.deepEqual(applyOpenAIToolChoice(tools, "none"), []) + assert.deepEqual(applyOpenAIToolChoice(tools, "auto"), tools) + assert.deepEqual( + applyOpenAIToolChoice(tools, { type: "function", function: { name: "b" } }).map((t) => t.name), + ["b"], + ) + assert.deepEqual(applyOpenAIToolChoice(tools, { type: "function", name: "a" }).map((t) => t.name), ["a"]) +}) + +test("parseAnthropicTools extracts tools with input_schema", () => { + const tools = parseAnthropicTools({ + tools: [{ name: "get_weather", description: "d", input_schema: { type: "object" } }], + }) + assert.equal(tools.length, 1) + assert.equal(tools[0].name, "get_weather") + assert.deepEqual(tools[0].parameters, { type: "object" }) +}) + +test("applyAnthropicToolChoice supports none and named tool", () => { + const tools = [{ name: "a" }, { name: "b" }] + assert.deepEqual(applyAnthropicToolChoice(tools, { type: "none" }), []) + assert.deepEqual(applyAnthropicToolChoice(tools, { type: "tool", name: "a" }).map((t) => t.name), ["a"]) + assert.deepEqual(applyAnthropicToolChoice(tools, { type: "auto" }), tools) +}) + +test("parseGeminiTools flattens functionDeclarations across tool groups", () => { + const tools = parseGeminiTools({ + tools: [ + { functionDeclarations: [{ name: "get_weather", description: "d", parameters: { type: "object" } }] }, + { functionDeclarations: [{ name: "get_time" }] }, + ], + }) + assert.equal(tools.length, 2) + assert.equal(tools[0].name, "get_weather") + assert.equal(tools[1].name, "get_time") +}) + +test("applyGeminiToolChoice supports NONE mode and allowedFunctionNames", () => { + const tools = [{ name: "a" }, { name: "b" }] + assert.deepEqual(applyGeminiToolChoice(tools, { functionCallingConfig: { mode: "NONE" } }), []) + assert.deepEqual( + applyGeminiToolChoice(tools, { functionCallingConfig: { mode: "ANY", allowedFunctionNames: ["b"] } }).map( + (t) => t.name, + ), + ["b"], + ) + assert.deepEqual(applyGeminiToolChoice(tools, undefined), tools) +}) + +// --------------------------------------------------------------------------- +// Unit: tool-call round-tripping in conversation history normalizers +// --------------------------------------------------------------------------- + +test("normalizeMessages renders prior OpenAI tool_calls and tool results as text", () => { + const messages = normalizeMessages([ + { role: "user", content: "What's the weather in NYC?" }, + { + role: "assistant", + content: null, + tool_calls: [{ id: "call_1", type: "function", function: { name: "get_weather", arguments: '{"city":"NYC"}' } }], + }, + { role: "tool", tool_call_id: "call_1", content: "Sunny, 72F" }, + ]) + + assert.equal(messages.length, 3) + assert.ok(messages[1].content.includes("get_weather")) + assert.ok(messages[1].content.includes('{"city":"NYC"}')) + assert.ok(messages[2].content.includes("get_weather")) + assert.ok(messages[2].content.includes("Sunny, 72F")) +}) + +test("normalizeAnthropicMessages renders prior tool_use and tool_result blocks as text", () => { + const messages = normalizeAnthropicMessages([ + { role: "user", content: "What's the weather in NYC?" }, + { + role: "assistant", + content: [{ type: "tool_use", id: "toolu_1", name: "get_weather", input: { city: "NYC" } }], + }, + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "toolu_1", content: "Sunny, 72F" }], + }, + ]) + + assert.equal(messages.length, 3) + assert.ok(messages[1].content.includes("get_weather")) + assert.ok(messages[1].content.includes("NYC")) + assert.ok(messages[2].content.includes("get_weather")) + assert.ok(messages[2].content.includes("Sunny, 72F")) +}) + +test("normalizeGeminiContents renders prior functionCall and functionResponse parts as text", () => { + const messages = normalizeGeminiContents([ + { role: "user", parts: [{ text: "What's the weather in NYC?" }] }, + { role: "model", parts: [{ functionCall: { name: "get_weather", args: { city: "NYC" } } }] }, + { role: "user", parts: [{ functionResponse: { name: "get_weather", response: { temp: "72F" } } }] }, + ]) + + assert.equal(messages.length, 3) + assert.ok(messages[1].content.includes("get_weather")) + assert.ok(messages[2].content.includes("get_weather")) + assert.ok(messages[2].content.includes("72F")) +}) + +test("normalizeResponseInput renders prior function_call and function_call_output items as text", () => { + const messages = normalizeResponseInput([ + { role: "user", content: "What's the weather in NYC?" }, + { type: "function_call", call_id: "call_1", name: "get_weather", arguments: '{"city":"NYC"}' }, + { type: "function_call_output", call_id: "call_1", output: "Sunny, 72F" }, + ]) + + assert.equal(messages.length, 3) + assert.ok(messages[1].content.includes("get_weather")) + assert.ok(messages[2].content.includes("get_weather")) + assert.ok(messages[2].content.includes("Sunny, 72F")) +}) + +// --------------------------------------------------------------------------- +// Integration: end-to-end tool calling via the dynamic MCP bridge +// --------------------------------------------------------------------------- + +function createToolCallClient({ toolName, toolArgs, callID = "call_1", finish = "tool_calls", providers } = {}) { + let capturedSlotName = null + + return { + app: { log: async () => {} }, + tool: { ids: async () => ({ data: [] }) }, + config: { + providers: async () => ({ + data: { + providers: providers ?? [{ id: "openai", models: { "gpt-4o": { id: "gpt-4o", name: "GPT-4o" } } }], + }, + }), + }, + mcp: { + disconnect: async () => { + throw new Error("not connected") + }, + add: async ({ body }) => { + capturedSlotName = body.name + assert.equal(body.config.type, "local") + assert.ok(Array.isArray(body.config.command)) + return { data: {} } + }, + }, + session: { + create: async () => ({ data: { id: "sess-tool-1" } }), + promptAsync: async () => {}, + abort: async () => ({ data: true }), + messages: async () => ({ + data: [ + { + role: "assistant", + tokens: { input: 5, output: 2, reasoning: 0, cache: { read: 0, write: 0 } }, + finish, + }, + ], + }), + }, + event: { + subscribe: async () => ({ + stream: (async function* () { + yield { + type: "message.part.updated", + properties: { + part: { + sessionID: "sess-tool-1", + type: "tool", + tool: `${capturedSlotName}_${toolName}`, + callID, + state: { status: "pending", input: toolArgs }, + }, + }, + } + })(), + }), + }, + } +} + +test("POST /v1/chat/completions returns tool_calls when the model calls a caller-supplied tool", async () => { + const client = createToolCallClient({ toolName: "get_weather", toolArgs: { city: "NYC" } }) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + messages: [{ role: "user", content: "What's the weather in NYC?" }], + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get the weather", + parameters: { type: "object", properties: { city: { type: "string" } } }, + }, + }, + ], + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.equal(body.choices[0].finish_reason, "tool_calls") + assert.equal(body.choices[0].message.content, null) + assert.equal(body.choices[0].message.tool_calls[0].function.name, "get_weather") + assert.deepEqual(JSON.parse(body.choices[0].message.tool_calls[0].function.arguments), { city: "NYC" }) + assert.equal(body.choices[0].message.tool_calls[0].id, "call_1") +}) + +test("POST /v1/chat/completions stream: true emits tool_calls delta and finish_reason", async () => { + const client = createToolCallClient({ toolName: "get_weather", toolArgs: { city: "NYC" } }) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + messages: [{ role: "user", content: "What's the weather in NYC?" }], + tools: [{ type: "function", function: { name: "get_weather" } }], + }), + }) + + const response = await handler(request) + const text = await response.text() + + assert.ok(text.includes('"tool_calls"')) + assert.ok(text.includes("get_weather")) + assert.ok(text.includes('"finish_reason":"tool_calls"')) +}) + +test("POST /v1/messages returns tool_use content block when the model calls a tool", async () => { + const client = createToolCallClient({ toolName: "get_weather", toolArgs: { city: "NYC" }, callID: "toolu_1" }) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + messages: [{ role: "user", content: "What's the weather in NYC?" }], + tools: [{ name: "get_weather", description: "Get weather", input_schema: { type: "object" } }], + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.equal(body.stop_reason, "tool_use") + assert.equal(body.content[0].type, "tool_use") + assert.equal(body.content[0].name, "get_weather") + assert.equal(body.content[0].id, "toolu_1") + assert.deepEqual(body.content[0].input, { city: "NYC" }) +}) + +test("POST /v1/messages stream: true emits a tool_use content block", async () => { + const client = createToolCallClient({ toolName: "get_weather", toolArgs: { city: "NYC" }, callID: "toolu_1" }) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + messages: [{ role: "user", content: "What's the weather in NYC?" }], + tools: [{ name: "get_weather" }], + }), + }) + + const response = await handler(request) + const text = await response.text() + + assert.ok(text.includes("tool_use")) + assert.ok(text.includes("get_weather")) + assert.ok(text.includes('"stop_reason":"tool_use"')) +}) + +test("POST /v1beta/models/:model:generateContent returns a functionCall part", async () => { + const client = createToolCallClient({ + toolName: "get_weather", + toolArgs: { city: "NYC" }, + providers: [{ id: "google", models: { "gemini-2.0-flash": { id: "gemini-2.0-flash" } } }], + }) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1beta/models/gemini-2.0-flash:generateContent", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "What's the weather in NYC?" }] }], + tools: [{ functionDeclarations: [{ name: "get_weather", description: "Get weather" }] }], + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.ok(body.candidates[0].content.parts[0].functionCall) + assert.equal(body.candidates[0].content.parts[0].functionCall.name, "get_weather") + assert.deepEqual(body.candidates[0].content.parts[0].functionCall.args, { city: "NYC" }) +}) + +test("POST /v1/responses returns a function_call output item", async () => { + const client = createToolCallClient({ toolName: "get_weather", toolArgs: { city: "NYC" } }) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + input: "What's the weather in NYC?", + tools: [{ type: "function", name: "get_weather", description: "Get weather" }], + }), + }) + + const response = await handler(request) + const body = await response.json() + + assert.equal(response.status, 200) + assert.equal(body.output[0].type, "function_call") + assert.equal(body.output[0].name, "get_weather") + assert.deepEqual(JSON.parse(body.output[0].arguments), { city: "NYC" }) + assert.equal(body.output_text, "") +}) + +test("POST /v1/responses stream: true emits function_call SSE events", async () => { + const client = createToolCallClient({ toolName: "get_weather", toolArgs: { city: "NYC" } }) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + input: "What's the weather in NYC?", + tools: [{ type: "function", name: "get_weather" }], + }), + }) + + const response = await handler(request) + const text = await response.text() + + assert.ok(text.includes("response.function_call_arguments.done")) + assert.ok(text.includes("get_weather")) + assert.ok(text.includes('"type":"function_call"')) +}) + +test("tool_choice: none disables tool calling even when tools are supplied", async () => { + const events = [{ type: "session.idle", properties: { sessionID: "sess-123" } }] + const client = createStreamingClient(events) + const handler = createProxyFetchHandler(client) + const request = new Request("http://127.0.0.1:4010/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4o", + stream: true, + messages: [{ role: "user", content: "hi" }], + tools: [{ type: "function", function: { name: "get_weather" } }], + tool_choice: "none", + }), + }) + + const response = await handler(request) + assert.equal(response.status, 200) + // No mcp/tool bridge client methods were exercised because callerTools resolved to []. + assert.equal(client.mcp, undefined) +}) diff --git a/mcp-tool-bridge.js b/mcp-tool-bridge.js new file mode 100644 index 0000000..98d53b0 --- /dev/null +++ b/mcp-tool-bridge.js @@ -0,0 +1,107 @@ +#!/usr/bin/env node +// Minimal MCP (Model Context Protocol) stdio server used internally by opencode-llm-proxy +// to expose a proxy caller's OpenAI/Anthropic/Gemini tool schemas to OpenCode as if they +// were real MCP tools. +// +// This process is spawned by OpenCode itself as a "local" MCP server (see index.js +// registerToolBridge()). It never actually executes anything: the proxy detects the +// resulting tool-call event on OpenCode's event stream and aborts the session before +// tools/call would matter, so the response returned here is just a harmless placeholder. +// +// Protocol: JSON-RPC 2.0 messages, newline-delimited, over stdin/stdout. +// stdout MUST only ever contain JSON-RPC messages - all diagnostics go to stderr. + +const toolsJson = process.env.OPENCODE_LLM_PROXY_BRIDGE_TOOLS ?? "[]" + +let tools = [] +try { + const parsed = JSON.parse(toolsJson) + if (Array.isArray(parsed)) tools = parsed +} catch (error) { + process.stderr.write(`opencode-llm-proxy bridge: failed to parse tool schemas: ${error}\n`) +} + +function send(message) { + process.stdout.write(JSON.stringify(message) + "\n") +} + +function respondResult(id, result) { + if (id === undefined || id === null) return + send({ jsonrpc: "2.0", id, result }) +} + +function respondError(id, code, message) { + if (id === undefined || id === null) return + send({ jsonrpc: "2.0", id, error: { code, message } }) +} + +function handleMessage(message) { + const { id, method, params } = message + + switch (method) { + case "initialize": { + respondResult(id, { + protocolVersion: params?.protocolVersion ?? "2024-11-05", + capabilities: { tools: {} }, + serverInfo: { name: "opencode-llm-proxy-bridge", version: "1.0.0" }, + }) + return + } + case "notifications/initialized": + // Notification, no response expected. + return + case "ping": { + respondResult(id, {}) + return + } + case "tools/list": { + respondResult(id, { + tools: tools.map((tool) => ({ + name: tool.name, + description: tool.description ?? "", + inputSchema: tool.parameters ?? { type: "object", properties: {} }, + })), + }) + return + } + case "tools/call": { + // Never actually reached in practice: the proxy aborts the OpenCode session as + // soon as it observes the tool-call part on the event stream, before this + // response would be consumed. Returned only as a safety net. + respondResult(id, { + content: [ + { + type: "text", + text: "(intercepted by opencode-llm-proxy; awaiting the external caller's tool result)", + }, + ], + }) + return + } + default: { + respondError(id, -32601, `Method not found: ${method}`) + } + } +} + +let buffer = "" +process.stdin.setEncoding("utf8") +process.stdin.on("data", (chunk) => { + buffer += chunk + let newlineIndex + while ((newlineIndex = buffer.indexOf("\n")) !== -1) { + const line = buffer.slice(0, newlineIndex).trim() + buffer = buffer.slice(newlineIndex + 1) + if (!line) continue + try { + const message = JSON.parse(line) + handleMessage(message) + } catch (error) { + process.stderr.write(`opencode-llm-proxy bridge: failed to parse message: ${error}\n`) + } + } +}) + +process.stdin.on("end", () => { + process.exit(0) +}) diff --git a/package.json b/package.json index 88d496b..f882471 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "opencode-llm-proxy", "version": "1.6.1", - "description": "Local AI gateway for OpenCode — use any model via OpenAI, Anthropic, or Gemini API format", + "description": "Local AI gateway for OpenCode with tool/function calling — use any model via OpenAI, Anthropic, or Gemini API format", "main": "index.js", "type": "module", "engines": { @@ -32,7 +32,15 @@ "ai-gateway", "model-router", "openrouter", - "bedrock" + "bedrock", + "tool-calling", + "function-calling", + "tools", + "mcp", + "model-context-protocol", + "coding-agent", + "ai-agent", + "agentic" ], "author": "KochC", "license": "MIT",