diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cbbeb35 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +node_modules +**/node_modules +.turbo +**/.turbo +dist +**/dist +.next +**/.next +coverage +**/coverage +*.log +.env +.env.* +**/.env +**/.env.* +!.env.example +!**/.env.example diff --git a/README.md b/README.md index 2bdecb8..e584097 100644 --- a/README.md +++ b/README.md @@ -188,13 +188,21 @@ Keep shared solutions generic and portable. Do not publish private repository na ## Private Local Mode -Use the MCP server without the hosted service: +Use the CLI and MCP server without the hosted service: ```bash CLANKER_MODE=local clanker mcp ``` -Local mode stores solutions in SQLite and does not call the hosted API. Keyword search is available locally; semantic search is not configured yet. Override the database path with `CLANKER_LOCAL_DB`. +Local mode stores solutions in SQLite and does not call the hosted API. The direct `clanker log`, `clanker search`, `clanker upvote`, and `clanker downvote` commands also use local storage when `CLANKER_MODE=local`. + +Keyword, semantic, and hybrid search are available locally by default. `clanker local embed` downloads/checks the default GGUF embedding model and embeds pending local solutions. Disable local semantic and hybrid search with `CLANKER_LOCAL_SEMANTIC=0`, `false`, or `off`. Override the database path with `CLANKER_LOCAL_DB` and the model path with `CLANKER_LOCAL_MODEL_PATH`. + +The Docker-isolated e2e check is available on demand: + +```bash +pnpm test:e2e:local +``` ## OpenClaw @@ -284,13 +292,17 @@ ClankerOverflow is available under the [MIT License](LICENSE). ## Environment Variables -| Variable | Purpose | Default | -| -------------------- | ------------------------------------------ | ------------------------------------------------- | -| `CLANKER_API_KEY` | Authenticate logging and voting | None | -| `CLANKER_SERVER_URL` | Override the API server | `https://api.clankeroverflow.com` | -| `CLANKER_WEB_URL` | Override links printed after logging | `https://clankeroverflow.com` | -| `CLANKER_MODE` | Set to `local` for offline SQLite MCP mode | `remote` | -| `CLANKER_LOCAL_DB` | Override the local SQLite database path | `~/.local/share/clankeroverflow/solutions.sqlite` | +| Variable | Purpose | Default | +| -------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------- | +| `CLANKER_API_KEY` | Authenticate hosted logging and voting | None | +| `CLANKER_SERVER_URL` | Override the API server | `https://api.clankeroverflow.com` | +| `CLANKER_WEB_URL` | Override links printed after hosted logging | `https://clankeroverflow.com` | +| `CLANKER_MODE` | Set to `local` for offline SQLite CLI/MCP mode | `remote` | +| `CLANKER_LOCAL_DB` | Override the local SQLite database path | `~/.local/share/clankeroverflow/solutions.sqlite` | +| `CLANKER_LOCAL_SEMANTIC` | Set to `0`, `false`, or `off` to disable local semantic and hybrid search | Enabled in local mode | +| `CLANKER_LOCAL_MODEL_PATH` | Override the local GGUF embedding model path | `$XDG_CACHE_HOME/clankeroverflow/models/...` | +| `CLANKER_LOCAL_MODEL_ID` | Override the local embedding model identifier | `bge-small-en-v1.5-q8_0` | +| `CLANKER_LOCAL_MODEL_DIMENSIONS` | Override local embedding dimensions | `384` | ## Deployment diff --git a/package.json b/package.json index 8bcd5b5..30070bd 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "dev:web": "turbo run dev --filter=web", "dev:server": "turbo run dev --filter=server", "test": "turbo run test", + "test:e2e:local": "tsx scripts/test-cli-local-e2e.ts", "db:push": "turbo run db:push --filter=@clankeroverflow/db", "db:generate": "turbo run db:generate --filter=@clankeroverflow/db", "db:migrate": "turbo run db:migrate --filter=@clankeroverflow/db", diff --git a/packages/cli/.claude-plugin/plugin.json b/packages/cli/.claude-plugin/plugin.json index 10b748a..08f74ca 100644 --- a/packages/cli/.claude-plugin/plugin.json +++ b/packages/cli/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "clankeroverflow", - "version": "1.0.22", + "version": "1.1.0", "description": "Search-first debugging memory for AI coding agents. Search prior fixes before fresh debugging, validate results, vote on tried solutions, and log verified reusable fixes.", "author": { "name": "ClankerOverflow", diff --git a/packages/cli/.codex-plugin/plugin.json b/packages/cli/.codex-plugin/plugin.json index ddf8ebf..d2f2771 100644 --- a/packages/cli/.codex-plugin/plugin.json +++ b/packages/cli/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "clankeroverflow", - "version": "1.0.22", + "version": "1.1.0", "description": "Search-first debugging memory for AI coding agents. Search prior fixes before fresh debugging, validate results, vote on tried solutions, and log verified reusable fixes.", "author": { "name": "ClankerOverflow", diff --git a/packages/cli/e2e/Dockerfile.local-mode b/packages/cli/e2e/Dockerfile.local-mode new file mode 100644 index 0000000..9fc39d1 --- /dev/null +++ b/packages/cli/e2e/Dockerfile.local-mode @@ -0,0 +1,30 @@ +FROM node:22-bookworm-slim + +ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 +ENV PNPM_HOME=/pnpm +ENV PATH=/pnpm:$PATH + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates g++ make python3 \ + && rm -rf /var/lib/apt/lists/* \ + && corepack enable + +WORKDIR /workspace + +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml turbo.json tsconfig.json ./ +COPY apps/server/package.json apps/server/package.json +COPY apps/web/package.json apps/web/package.json +COPY packages/api/package.json packages/api/package.json +COPY packages/auth/package.json packages/auth/package.json +COPY packages/cli/package.json packages/cli/package.json +COPY packages/config/package.json packages/config/package.json +COPY packages/db/package.json packages/db/package.json +COPY packages/env/package.json packages/env/package.json +COPY packages/infra/package.json packages/infra/package.json +RUN pnpm install --frozen-lockfile + +COPY . . + +RUN pnpm --filter @clankeroverflow/cli run build + +CMD ["node", "packages/cli/e2e/local-mode.mjs"] diff --git a/packages/cli/e2e/local-mode.mjs b/packages/cli/e2e/local-mode.mjs new file mode 100644 index 0000000..cbc34c3 --- /dev/null +++ b/packages/cli/e2e/local-mode.mjs @@ -0,0 +1,275 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const cliPath = join(root, "packages/cli/dist/index.mjs"); + +const fixtures = { + vite: { + problem: "Vite dev server exits with EADDRINUSE when port 5173 is already bound", + solution: "Find the process that owns the port and stop it, or start Vite on a different port.", + tags: "vite,dev-server,ports", + }, + playwright: { + problem: "Playwright browser install is missing on Debian CI", + solution: + "Run playwright install --with-deps chromium so browsers and operating system libraries exist before tests.", + tags: "playwright,ci,browser", + }, + prisma: { + problem: "Prisma migration shadow database permission denied", + solution: + "Grant create database permission for the test user or configure a dedicated shadow database URL.", + tags: "prisma,postgres,migrations", + }, +}; + +function logStep(message) { + console.log(`[local-mode-e2e] ${message}`); +} + +function textFromTool(result) { + return (result.content ?? []) + .filter((entry) => entry.type === "text") + .map((entry) => entry.text) + .join("\n"); +} + +function firstProblem(output) { + return output.match(/^# Problem: (?.+?) \(Score: /m)?.groups?.problem ?? ""; +} + +function assertTopProblem(output, expectedProblem, label) { + assert.equal( + firstProblem(output), + expectedProblem, + `${label} should return the expected top problem.\n\n${output}`, + ); +} + +async function runCli(args, env) { + const result = await runProcess(process.execPath, [cliPath, ...args], { + cwd: root, + env, + }); + return result.stdout; +} + +async function runProcess(command, args, options) { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + + const exitCode = await new Promise((resolveProcess, rejectProcess) => { + child.on("error", rejectProcess); + child.on("exit", (code) => resolveProcess(code ?? 0)); + }); + + if (exitCode !== 0) { + throw new Error( + [ + `Command failed with exit code ${exitCode}: ${command} ${args.join(" ")}`, + stderr && `stderr:\n${stderr}`, + stdout && `stdout:\n${stdout}`, + ] + .filter(Boolean) + .join("\n\n"), + ); + } + + return { stdout, stderr }; +} + +async function logDirectSolution(env, fixture) { + const stdout = await runCli( + ["log", "--problem", fixture.problem, "--solution", fixture.solution, "--tags", fixture.tags], + env, + ); + const id = stdout.match(/[0-9a-f-]{36}/)?.[0]; + assert.ok(id, `direct log output should contain a local UUID.\n\n${stdout}`); + return id; +} + +async function verifyDirectCli(env) { + logStep("checking native semantic dependencies import"); + await import("sqlite-vec"); + await import("sqlite-lembed"); + + logStep("downloading or checking the local embedding model"); + const embedOutput = await runCli(["local", "embed"], env); + assert.match(embedOutput, /Local embeddings ready/); + + logStep("logging direct CLI fixture solutions"); + await logDirectSolution(env, fixtures.vite); + await logDirectSolution(env, fixtures.playwright); + await logDirectSolution(env, fixtures.prisma); + + logStep("checking local semantic status after direct logs"); + const status = JSON.parse(await runCli(["local", "status", "--json"], env)); + assert.equal(status.mode, "local"); + assert.equal(status.semantic.enabled, true); + assert.equal(status.semantic.totalSolutions, 3); + assert.equal(status.semantic.embeddedSolutions, 3); + assert.equal(status.semantic.pendingEmbeddings, 0); + assert.equal(status.semantic.staleEmbeddings, 0); + assert.equal(status.semantic.modelValid, true); + assert.equal(status.semantic.sqliteVecAvailable, true); + assert.equal(status.semantic.embedderAvailable, true); + + logStep("verifying direct keyword search"); + const keyword = await runCli(["search", "EADDRINUSE", "--mode", "keyword", "--limit", "1"], env); + assertTopProblem(keyword, fixtures.vite.problem, "direct keyword search"); + + logStep("verifying direct semantic search"); + const semanticQuery = "address already occupied during frontend startup"; + const semantic = await runCli( + ["search", semanticQuery, "--mode", "semantic", "--limit", "1"], + env, + ); + assertTopProblem(semantic, fixtures.vite.problem, "direct semantic search"); + + logStep("verifying direct hybrid search"); + const hybrid = await runCli(["search", semanticQuery, "--mode", "hybrid", "--limit", "1"], env); + assertTopProblem(hybrid, fixtures.vite.problem, "direct hybrid search"); + + logStep("verifying direct auto fallback to hybrid"); + const auto = await runCli(["search", semanticQuery, "--limit", "1"], env); + assert.match(auto, /Search attempts: keyword returned 0; hybrid returned 1\./); + assertTopProblem(auto, fixtures.vite.problem, "direct auto search"); +} + +async function verifyMcp(env) { + logStep("starting MCP server over stdio"); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [cliPath, "mcp"], + cwd: root, + env, + stderr: "pipe", + }); + const stderrChunks = []; + transport.stderr?.setEncoding("utf8"); + transport.stderr?.on("data", (chunk) => { + stderrChunks.push(chunk); + }); + + const client = new Client({ name: "clankeroverflow-local-e2e", version: "1.0.0" }); + await client.connect(transport, { timeout: 120_000 }); + + try { + logStep("logging an MCP fixture solution"); + const logResult = await client.callTool( + { + name: "log_solution", + arguments: { + problem: "Node test runner cannot resolve workspace package exports", + solution: + "Build the referenced workspace package first so package exports point at existing dist files.", + tags: "node,pnpm,workspace", + }, + }, + undefined, + { timeout: 120_000 }, + ); + assert.match(textFromTool(logResult), /Solution logged locally: [0-9a-f-]{36}/); + + logStep("checking MCP local status"); + const statusResult = await client.callTool( + { name: "clanker_status", arguments: {} }, + undefined, + { timeout: 120_000 }, + ); + assert.match(textFromTool(statusResult), /ClankerOverflow mode: local/); + assert.equal(statusResult.structuredContent?.mode, "local"); + assert.equal(statusResult.structuredContent?.semantic?.totalSolutions, 4); + assert.equal(statusResult.structuredContent?.semantic?.embeddedSolutions, 4); + assert.equal(statusResult.structuredContent?.semantic?.pendingEmbeddings, 0); + assert.equal(statusResult.structuredContent?.semantic?.modelValid, true); + assert.equal(statusResult.structuredContent?.semantic?.sqliteVecAvailable, true); + assert.equal(statusResult.structuredContent?.semantic?.embedderAvailable, true); + + logStep("verifying MCP semantic search"); + const semanticResult = await client.callTool( + { + name: "search_solutions", + arguments: { + query: "browser dependencies unavailable in linux automation", + mode: "semantic", + limit: 1, + }, + }, + undefined, + { timeout: 120_000 }, + ); + assertTopProblem( + textFromTool(semanticResult), + fixtures.playwright.problem, + "MCP semantic search", + ); + + logStep("verifying MCP auto fallback to hybrid"); + const autoResult = await client.callTool( + { + name: "search_solutions", + arguments: { + query: "address already occupied during frontend startup", + limit: 1, + }, + }, + undefined, + { timeout: 120_000 }, + ); + const autoText = textFromTool(autoResult); + assert.match(autoText, /Search attempts: keyword returned 0; hybrid returned 1\./); + assertTopProblem(autoText, fixtures.vite.problem, "MCP auto search"); + } catch (error) { + const stderr = stderrChunks.join(""); + if (stderr) console.error(stderr); + throw error; + } finally { + await client.close(); + } +} + +const tempRoot = await mkdtemp(join(tmpdir(), "clanker-local-e2e-")); + +try { + const home = join(tempRoot, "home"); + await mkdir(home, { recursive: true }); + const env = { + ...process.env, + HOME: home, + NO_COLOR: "1", + XDG_CACHE_HOME: process.env.XDG_CACHE_HOME || join(tempRoot, "cache"), + CLANKER_MODE: "local", + CLANKER_LOCAL_DB: join(tempRoot, "solutions.sqlite"), + CLANKER_SERVER_URL: "http://127.0.0.1:9", + CLANKER_WEB_URL: "http://127.0.0.1:9", + CLANKER_API_KEY: "", + }; + + await verifyDirectCli(env); + await verifyMcp(env); + logStep("passed"); +} finally { + await rm(tempRoot, { recursive: true, force: true }); +} diff --git a/packages/cli/openclaw.plugin.json b/packages/cli/openclaw.plugin.json index cbda49f..203ec3f 100644 --- a/packages/cli/openclaw.plugin.json +++ b/packages/cli/openclaw.plugin.json @@ -2,7 +2,7 @@ "id": "@bernoussama/clankeroverflow", "name": "ClankerOverflow", "description": "Search-first debugging memory for AI coding agents. Search prior fixes before fresh debugging, validate results, vote on tried solutions, and log verified reusable fixes.", - "version": "1.0.22", + "version": "1.1.0", "configSchema": { "type": "object", "additionalProperties": false diff --git a/packages/cli/package.json b/packages/cli/package.json index df63531..98becd7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@clankeroverflow/cli", - "version": "1.0.22", + "version": "1.1.0", "description": "ClankerOverflow CLI for logging and searching AI agent solutions", "license": "MIT", "repository": { diff --git a/packages/cli/skills/clankeroverflow-mcp/SKILL.md b/packages/cli/skills/clankeroverflow-mcp/SKILL.md index 42b4347..313d6cf 100644 --- a/packages/cli/skills/clankeroverflow-mcp/SKILL.md +++ b/packages/cli/skills/clankeroverflow-mcp/SKILL.md @@ -83,9 +83,12 @@ Use this only after verification. - Users can opt into private offline storage with `CLANKER_MODE=local clanker mcp`. - Local mode stores solutions in SQLite and never calls the hosted API. +- The direct `clanker log`, `clanker search`, `clanker upvote`, and `clanker downvote` commands also use local storage when `CLANKER_MODE=local`. - `CLANKER_LOCAL_DB` can override the SQLite path; otherwise the server uses the OS default data directory. - In local mode, all four tools work without `CLANKER_API_KEY`. -- Treat `semantic` search as unavailable in local mode unless the server reports otherwise; auto mode uses local keyword search. +- Local semantic and hybrid search are enabled by default with the configured GGUF model. Run `clanker local embed` to download/check the default model and embed pending local solutions. +- Set `CLANKER_LOCAL_SEMANTIC=0`, `false`, or `off` to disable local semantic and hybrid search. +- Treat `semantic` search as unavailable in local mode only when the server reports semantic search is disabled or unhealthy. ## Response style diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index c5459d0..fcc35be 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -1,7 +1,42 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { afterEach, beforeEach, describe, expect, test, vi, type MockInstance } from "vitest"; import { createProgram } from "./index"; import pc from "picocolors"; +async function withLocalCliEnv(run: (dbPath: string) => Promise) { + const previousMode = process.env.CLANKER_MODE; + const previousDb = process.env.CLANKER_LOCAL_DB; + const previousSemantic = process.env.CLANKER_LOCAL_SEMANTIC; + const dir = mkdtempSync(join(tmpdir(), "clanker-cli-local-")); + + try { + process.env.CLANKER_MODE = "local"; + process.env.CLANKER_LOCAL_DB = join(dir, "solutions.sqlite"); + process.env.CLANKER_LOCAL_SEMANTIC = "0"; + return await run(process.env.CLANKER_LOCAL_DB); + } finally { + if (previousMode === undefined) { + delete process.env.CLANKER_MODE; + } else { + process.env.CLANKER_MODE = previousMode; + } + if (previousDb === undefined) { + delete process.env.CLANKER_LOCAL_DB; + } else { + process.env.CLANKER_LOCAL_DB = previousDb; + } + if (previousSemantic === undefined) { + delete process.env.CLANKER_LOCAL_SEMANTIC; + } else { + process.env.CLANKER_LOCAL_SEMANTIC = previousSemantic; + } + rmSync(dir, { recursive: true, force: true }); + } +} + describe("CLI", () => { let consoleLogMock: MockInstance; let consoleErrorMock: MockInstance; @@ -79,6 +114,28 @@ describe("CLI", () => { ` Solution logged: ${pc.cyan(pc.underline("https://clankeroverflow.com/solution/123"))}`, ); }); + + test("logs locally without calling the hosted API", async () => { + await withLocalCliEnv(async () => { + const program = createProgram(); + await program.parseAsync([ + "node", + "test", + "log", + "--problem", + "Local SQLite WAL busy error", + "--solution", + "Close stale readers before retrying the write transaction", + "--tags", + "sqlite,local", + ]); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(consoleLogMock).toHaveBeenCalledWith( + expect.stringContaining("Solution logged locally:"), + ); + }); + }); }); describe("search command", () => { @@ -200,6 +257,42 @@ describe("CLI", () => { expect(allOutput).not.toContain("\x07"); expect(allOutput).toContain("Fake Error"); }); + + test("searches local solutions without calling the hosted API", async () => { + await withLocalCliEnv(async () => { + const logProgram = createProgram(); + await logProgram.parseAsync([ + "node", + "test", + "log", + "--problem", + "Local sqlite vector extension fails to load", + "--solution", + "Install the Node native dependency inside the same runtime image", + "--tags", + "sqlite-vec,docker", + ]); + consoleLogMock.mockClear(); + + const searchProgram = createProgram(); + await searchProgram.parseAsync([ + "node", + "test", + "search", + "sqlite-vec", + "--mode", + "keyword", + "--limit", + "1", + ]); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(consoleLogMock).toHaveBeenCalledWith(expect.stringContaining("sqlite vector")); + expect(consoleLogMock).toHaveBeenCalledWith( + expect.stringContaining("Tags: sqlite-vec,docker"), + ); + }); + }); }); describe("vote commands", () => { @@ -230,6 +323,35 @@ describe("CLI", () => { pc.red(pc.bold("▼ Downvoted")) + ` solution ${pc.cyan("123")}`, ); }); + + test("votes locally without calling the hosted API", async () => { + await withLocalCliEnv(async () => { + const logProgram = createProgram(); + await logProgram.parseAsync([ + "node", + "test", + "log", + "--problem", + "Local vote target", + "--solution", + "Use the local backend", + ]); + const logged = consoleLogMock.mock.calls + .map((call) => call.join("")) + .find((line) => line.includes("Solution logged locally:")); + const id = logged?.match(/[0-9a-f-]{36}/)?.[0]; + expect(id).toBeDefined(); + consoleLogMock.mockClear(); + + const upvoteProgram = createProgram(); + await upvoteProgram.parseAsync(["node", "test", "upvote", id!]); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(consoleLogMock).toHaveBeenCalledWith( + pc.green(pc.bold("▲ Upvoted")) + ` solution ${pc.cyan(id!)}`, + ); + }); + }); }); describe("mcp command", () => { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 2dabea1..7e16ef3 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,14 +1,13 @@ #!/usr/bin/env node import { Command } from "commander"; -import { createTRPCClient, httpBatchLink } from "@trpc/client"; -import type { AppRouter } from "@clankeroverflow/api/routers/index"; import fs from "fs/promises"; import path from "path"; import packageJson from "../package.json"; import { searchWithAutoFallback } from "./mcp/auto-search.js"; -import type { ConcreteSearchMode, SearchMode } from "./mcp/backend.js"; +import type { SearchMode } from "./mcp/backend.js"; import { resolveConfig } from "./mcp/config.js"; +import { createSolutionBackend } from "./mcp/create-backend.js"; import { startMcpServer } from "./mcp/server.js"; import { formatSearchResults } from "./mcp/format.js"; import { LocalBackend } from "./mcp/local-backend.js"; @@ -58,31 +57,6 @@ function formatDoctor(checks: Array<{ name: string; ok: boolean; detail: string ].join("\n"); } -// Allow overriding via environment variables -const SERVER_URL = process.env.CLANKER_SERVER_URL || "https://api.clankeroverflow.com"; - -function getApiKey() { - return process.env.CLANKER_API_KEY || ""; -} - -const trpc = createTRPCClient({ - links: [ - httpBatchLink({ - url: `${SERVER_URL}/trpc`, - fetch(url, options) { - // Miniflare/Workers dev can intermittently fail when Node fetch is passed an AbortSignal. - // tRPC always supplies a signal, so strip it for CLI stability. - const { signal: _signal, ...rest } = options ?? {}; - return fetch(url, rest); - }, - headers() { - const apiKey = getApiKey(); - return apiKey ? { "x-clanker-api-key": apiKey } : {}; - }, - }), - ], -}); - export function createProgram(options: CreateProgramOptions = {}) { const program = new Command(); const runMcpServer = options.startMcpServer ?? startMcpServer; @@ -130,17 +104,25 @@ export function createProgram(options: CreateProgramOptions = {}) { process.exit(1); } - const result = await trpc.solutions.log.mutate({ + const config = resolveConfig(); + const backend = createSolutionBackend(config); + const result = await backend.log({ problem: options.problem, solution: solutionText, tags: options.tags, }); - const webUrl = process.env.CLANKER_WEB_URL || "https://clankeroverflow.com"; - console.log( - pc.green(pc.bold("✔ Success!")) + - ` Solution logged: ${pc.cyan(pc.underline(`${webUrl}/solution/${result.id}`))}`, - ); + if (config.mode === "local") { + console.log( + pc.green(pc.bold("✔ Success!")) + ` Solution logged locally: ${pc.cyan(result.id)}`, + ); + if (result.warning) console.log(pc.yellow(result.warning)); + } else { + console.log( + pc.green(pc.bold("✔ Success!")) + + ` Solution logged: ${pc.cyan(pc.underline(`${config.webUrl}/solution/${result.id}`))}`, + ); + } } catch (error: any) { console.error(pc.red(pc.bold("✖ Error logging solution:"))); console.error(pc.red(error.message || error)); @@ -175,19 +157,19 @@ export function createProgram(options: CreateProgramOptions = {}) { process.exit(1); } - const searchResult = await searchWithAutoFallback( - { - search: (input: { query: string; limit: number; mode: ConcreteSearchMode }) => - trpc.solutions.search.query(input), - }, - { - query, - limit, - mode, - allowHybridFallback: Boolean(getApiKey()), - fallbackUnavailableReason: "CLANKER_API_KEY is required for hosted hybrid fallback", - }, - ); + const config = resolveConfig(); + const backend = createSolutionBackend(config); + const searchResult = await searchWithAutoFallback(backend, { + query, + limit, + mode, + allowHybridFallback: + config.mode === "local" ? config.localSemantic.enabled : Boolean(config.apiKey), + fallbackUnavailableReason: + config.mode === "local" + ? "local semantic search is not configured" + : "CLANKER_API_KEY is required for hosted hybrid fallback", + }); const sanitized = searchResult.results.map((result) => ({ id: result.id, @@ -215,7 +197,8 @@ export function createProgram(options: CreateProgramOptions = {}) { .argument("", "The solution ID") .action(async (id) => { try { - await trpc.solutions.vote.mutate({ id, isUpvote: true }); + const backend = createSolutionBackend(resolveConfig()); + await backend.vote({ id, isUpvote: true }); console.log(pc.green(pc.bold("▲ Upvoted")) + ` solution ${pc.cyan(id)}`); } catch (error: any) { console.error(pc.red(pc.bold("✖ Error upvoting solution:"))); @@ -230,7 +213,8 @@ export function createProgram(options: CreateProgramOptions = {}) { .argument("", "The solution ID") .action(async (id) => { try { - await trpc.solutions.vote.mutate({ id, isUpvote: false }); + const backend = createSolutionBackend(resolveConfig()); + await backend.vote({ id, isUpvote: false }); console.log(pc.red(pc.bold("▼ Downvoted")) + ` solution ${pc.cyan(id)}`); } catch (error: any) { console.error(pc.red(pc.bold("✖ Error downvoting solution:"))); @@ -299,9 +283,7 @@ export function createProgram(options: CreateProgramOptions = {}) { { name: "local semantic enabled", ok: status.enabled, - detail: status.enabled - ? "enabled" - : "set CLANKER_LOCAL_SEMANTIC=1 or run setup --local-semantic", + detail: status.enabled ? "enabled" : "disabled by CLANKER_LOCAL_SEMANTIC=0/false/off", }, { name: "sqlite-vec", diff --git a/packages/cli/src/mcp/config.test.ts b/packages/cli/src/mcp/config.test.ts new file mode 100644 index 0000000..aa88732 --- /dev/null +++ b/packages/cli/src/mcp/config.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "vitest"; + +import { resolveConfig } from "./config"; + +describe("MCP config", () => { + test("enables local semantic search by default in local mode", () => { + expect(resolveConfig({ CLANKER_MODE: "local" }).localSemantic.enabled).toBe(true); + }); + + test("allows local semantic search to be disabled explicitly", () => { + for (const value of ["0", "false", "off"]) { + expect( + resolveConfig({ CLANKER_MODE: "local", CLANKER_LOCAL_SEMANTIC: value }).localSemantic + .enabled, + ).toBe(false); + } + }); + + test("keeps semantic search disabled outside local mode", () => { + expect(resolveConfig({ CLANKER_LOCAL_SEMANTIC: "1" }).localSemantic.enabled).toBe(false); + }); +}); diff --git a/packages/cli/src/mcp/config.ts b/packages/cli/src/mcp/config.ts index bcb6419..991073e 100644 --- a/packages/cli/src/mcp/config.ts +++ b/packages/cli/src/mcp/config.ts @@ -36,6 +36,12 @@ function expandHome(path: string) { return path; } +function localSemanticEnabled(env: NodeJS.ProcessEnv, mode: ClankerMode) { + if (mode !== "local") return false; + const value = env.CLANKER_LOCAL_SEMANTIC?.toLowerCase(); + return value !== "0" && value !== "false" && value !== "off"; +} + export function resolveConfig(env: NodeJS.ProcessEnv = process.env): ServerConfig { const mode = env.CLANKER_MODE === "local" ? "local" : "remote"; const localDbPath = resolve(expandHome(env.CLANKER_LOCAL_DB || defaultLocalDbPath())); @@ -45,7 +51,7 @@ export function resolveConfig(env: NodeJS.ProcessEnv = process.env): ServerConfi mode, localDbPath, localSemantic: { - enabled: env.CLANKER_LOCAL_SEMANTIC === "1" || env.CLANKER_LOCAL_SEMANTIC === "true", + enabled: localSemanticEnabled(env, mode), modelId: env.CLANKER_LOCAL_MODEL_ID || DEFAULT_LOCAL_MODEL_ID, modelPath, dimensions: Number(env.CLANKER_LOCAL_MODEL_DIMENSIONS || DEFAULT_LOCAL_MODEL_DIMENSIONS), diff --git a/packages/cli/src/mcp/create-backend.ts b/packages/cli/src/mcp/create-backend.ts new file mode 100644 index 0000000..52f46df --- /dev/null +++ b/packages/cli/src/mcp/create-backend.ts @@ -0,0 +1,15 @@ +import type { SolutionBackend } from "./backend"; +import { resolveConfig, type ServerConfig } from "./config"; +import { LocalBackend } from "./local-backend"; +import { RemoteBackend } from "./remote-backend"; + +export function createSolutionBackend(config: ServerConfig = resolveConfig()): SolutionBackend { + if (config.mode === "local") { + return new LocalBackend(config.localDbPath, { semantic: config.localSemantic }); + } + + return new RemoteBackend({ + serverUrl: config.serverUrl, + apiKey: config.apiKey, + }); +} diff --git a/packages/cli/src/mcp/local-backend.test.ts b/packages/cli/src/mcp/local-backend.test.ts index 799fa8c..8689836 100644 --- a/packages/cli/src/mcp/local-backend.test.ts +++ b/packages/cli/src/mcp/local-backend.test.ts @@ -158,6 +158,34 @@ describe("CLI local MCP backend", () => { expect(results[0]!.problem).toBe("OAuth callback timeout"); }); + test("status loads sqlite-vec before inspecting vector rows from an existing database", async () => { + const semantic: LocalSemanticConfig = { + enabled: true, + modelId: "test-model", + modelPath, + dimensions: 4, + }; + const firstBackend = new LocalBackend(dbPath, { + semantic, + embedder: { embed: () => vector([1, 0, 0, 0]) }, + }); + await firstBackend.log({ + problem: "OAuth callback timeout", + solution: "Keep waitUntil tasks alive", + tags: "auth", + }); + + const freshBackend = new LocalBackend(dbPath, { + semantic, + embedder: { embed: () => vector([1, 0, 0, 0]) }, + }); + + await expect(freshBackend.status()).resolves.toMatchObject({ + embeddedSolutions: 1, + pendingEmbeddings: 0, + }); + }); + test("embedding fingerprint changes when model file contents change", () => { const semantic: LocalSemanticConfig = { enabled: true, diff --git a/packages/cli/src/mcp/local-semantic.ts b/packages/cli/src/mcp/local-semantic.ts index 7fc2705..7ebb949 100644 --- a/packages/cli/src/mcp/local-semantic.ts +++ b/packages/cli/src/mcp/local-semantic.ts @@ -300,11 +300,11 @@ export function insertEmbedding( export function getSolutionsNeedingEmbedding( db: LocalDb, config: LocalSemanticConfig, - options: { limit?: number; fingerprint?: string } = {}, + options: { limit?: number; fingerprint?: string; includeVectorRows?: boolean } = {}, ) { ensureLocalSemanticSchema(db); const fingerprint = options.fingerprint ?? statusEmbeddingFingerprint(config); - const hasVecTable = hasSolutionVecTable(db); + const hasVecTable = options.includeVectorRows ?? hasSolutionVecTable(db); const rows = db .prepare( `SELECT solution.id, solution.problem, solution.solution, solution.tags, @@ -346,17 +346,6 @@ export async function getLocalSemanticStatus(db: LocalDb, config: LocalSemanticC const totalSolutions = ( db.prepare("SELECT COUNT(*) AS count FROM solution").get() as { count: number } ).count; - const pendingSolutions = getSolutionsNeedingEmbedding(db, config, { fingerprint }); - const embeddedSolutions = totalSolutions - pendingSolutions.length; - const staleEmbeddings = ( - db - .prepare( - `SELECT COUNT(*) AS count - FROM solution_embedding - WHERE model != ? OR embedding_fingerprint != ? OR dimensions != ?`, - ) - .get(config.modelId, fingerprint, config.dimensions) as { count: number } - ).count; const hasVecTable = hasSolutionVecTable(db); let sqliteVecAvailable = true; @@ -368,6 +357,21 @@ export async function getLocalSemanticStatus(db: LocalDb, config: LocalSemanticC sqliteVecError = error instanceof Error ? error.message : String(error); } + const pendingSolutions = getSolutionsNeedingEmbedding(db, config, { + fingerprint, + includeVectorRows: hasVecTable && sqliteVecAvailable, + }); + const embeddedSolutions = totalSolutions - pendingSolutions.length; + const staleEmbeddings = ( + db + .prepare( + `SELECT COUNT(*) AS count + FROM solution_embedding + WHERE model != ? OR embedding_fingerprint != ? OR dimensions != ?`, + ) + .get(config.modelId, fingerprint, config.dimensions) as { count: number } + ).count; + let embedderAvailable = true; let embedderError: string | undefined; try { diff --git a/packages/cli/src/mcp/server.test.ts b/packages/cli/src/mcp/server.test.ts index efaf907..6410bfe 100644 --- a/packages/cli/src/mcp/server.test.ts +++ b/packages/cli/src/mcp/server.test.ts @@ -282,11 +282,13 @@ describe("CLI MCP server", () => { test("local semantic search returns not-configured message without fetch", async () => { const previousMode = process.env.CLANKER_MODE; const previousDb = process.env.CLANKER_LOCAL_DB; + const previousSemantic = process.env.CLANKER_LOCAL_SEMANTIC; const dir = mkdtempSync(join(tmpdir(), "clanker-mcp-local-server-")); try { process.env.CLANKER_MODE = "local"; process.env.CLANKER_LOCAL_DB = join(dir, "solutions.sqlite"); + process.env.CLANKER_LOCAL_SEMANTIC = "0"; const localServer = createMcpServer(); const [localClientTransport, localServerTransport] = InMemoryTransport.createLinkedPair(); @@ -314,6 +316,11 @@ describe("CLI MCP server", () => { } else { process.env.CLANKER_LOCAL_DB = previousDb; } + if (previousSemantic === undefined) { + delete process.env.CLANKER_LOCAL_SEMANTIC; + } else { + process.env.CLANKER_LOCAL_SEMANTIC = previousSemantic; + } rmSync(dir, { recursive: true, force: true }); } }); diff --git a/packages/cli/src/mcp/server.ts b/packages/cli/src/mcp/server.ts index 49e784a..b9d0594 100644 --- a/packages/cli/src/mcp/server.ts +++ b/packages/cli/src/mcp/server.ts @@ -7,9 +7,9 @@ import packageJson from "../../package.json"; import { searchWithAutoFallback } from "./auto-search.js"; import type { SolutionBackend } from "./backend.js"; import { resolveConfig } from "./config.js"; +import { createSolutionBackend } from "./create-backend.js"; import { formatSearchResults } from "./format.js"; import { LocalBackend, LocalSemanticSearchNotConfiguredError } from "./local-backend.js"; -import { RemoteBackend } from "./remote-backend.js"; const logger = new McpLogger({ name: packageJson.name }); @@ -27,13 +27,7 @@ const SERVER_INSTRUCTIONS = [ export function createMcpServer() { const config = resolveConfig(); - const backend: SolutionBackend = - config.mode === "local" - ? new LocalBackend(config.localDbPath, { semantic: config.localSemantic }) - : new RemoteBackend({ - serverUrl: config.serverUrl, - apiKey: config.apiKey, - }); + const backend: SolutionBackend = createSolutionBackend(config); logger.debug("created backend", { mode: config.mode }); const server = new McpServer( diff --git a/scripts/test-cli-local-e2e.ts b/scripts/test-cli-local-e2e.ts new file mode 100644 index 0000000..3dde11a --- /dev/null +++ b/scripts/test-cli-local-e2e.ts @@ -0,0 +1,33 @@ +import { spawn } from "node:child_process"; + +const IMAGE = process.env.CLANKER_LOCAL_E2E_IMAGE || "clankeroverflow-cli-local-e2e:latest"; +const MODEL_VOLUME = + process.env.CLANKER_LOCAL_E2E_MODEL_VOLUME || "clankeroverflow_cli_e2e_model_cache"; + +async function run(cmd: string[]) { + const proc = spawn(cmd[0]!, cmd.slice(1), { + stdio: "inherit", + }); + + const exitCode = await new Promise((resolveProcess, rejectProcess) => { + proc.on("error", rejectProcess); + proc.on("exit", resolveProcess); + }); + + if (exitCode !== 0) { + throw new Error(`Command failed with exit code ${exitCode}: ${cmd.join(" ")}`); + } +} + +await run(["docker", "build", "-f", "packages/cli/e2e/Dockerfile.local-mode", "-t", IMAGE, "."]); +await run(["docker", "volume", "create", MODEL_VOLUME]); +await run([ + "docker", + "run", + "--rm", + "-e", + "XDG_CACHE_HOME=/model-cache", + "-v", + `${MODEL_VOLUME}:/model-cache`, + IMAGE, +]);