From 7a77cf2690b8460a25389d782f4aa2360438ec0c Mon Sep 17 00:00:00 2001 From: Oussama Bernou Date: Sun, 21 Jun 2026 16:37:02 +0100 Subject: [PATCH] fix chunked local embeddings --- README.md | 4 +- packages/cli/.claude-plugin/plugin.json | 2 +- packages/cli/.codex-plugin/plugin.json | 2 +- packages/cli/e2e/local-mode.mjs | 100 +++++++++++- packages/cli/openclaw.plugin.json | 2 +- packages/cli/package.json | 2 +- .../cli/skills/clankeroverflow-cli/SKILL.md | 7 + .../cli/skills/clankeroverflow-mcp/SKILL.md | 3 +- packages/cli/src/index.test.ts | 154 +++++++++++++++++- packages/cli/src/index.ts | 114 +++++++++---- packages/cli/src/mcp/local-backend.test.ts | 56 +++++++ packages/cli/src/mcp/local-semantic.test.ts | 56 +++++++ packages/cli/src/mcp/local-semantic.ts | 95 ++++++++++- 13 files changed, 547 insertions(+), 50 deletions(-) create mode 100644 packages/cli/src/mcp/local-semantic.test.ts diff --git a/README.md b/README.md index 9a26bf6..5b4726f 100644 --- a/README.md +++ b/README.md @@ -194,9 +194,9 @@ Use the CLI and MCP server without the hosted service: CLANKER_MODE=local clanker mcp ``` -Local mode stores solutions in SQLite and does not call the hosted API. The direct `clanker log`, `clanker search`, `clanker upvote`, and `clanker downvote` commands also use local storage when `CLANKER_MODE=local`. +Local mode stores solutions in SQLite and does not call the hosted API. Use `clanker local search ""` to explicitly search the local database without changing your shell environment. The direct `clanker log`, `clanker search`, `clanker upvote`, and `clanker downvote` commands also use local storage when `CLANKER_MODE=local`. -Keyword, semantic, and hybrid search are available locally by default. `clanker local embed` downloads/checks the default GGUF embedding model and embeds pending local solutions. Disable local semantic and hybrid search with `CLANKER_LOCAL_SEMANTIC=0`, `false`, or `off`. Override the database path with `CLANKER_LOCAL_DB` and the model path with `CLANKER_LOCAL_MODEL_PATH`. +Keyword, semantic, and hybrid search are available locally by default. `clanker local embed` downloads/checks the default GGUF embedding model and repairs pending or stale local embeddings. Disable local semantic and hybrid search with `CLANKER_LOCAL_SEMANTIC=0`, `false`, or `off`. Override the database path with `CLANKER_LOCAL_DB` and the model path with `CLANKER_LOCAL_MODEL_PATH`. The Docker-isolated e2e check runs the local-mode suite against Node 22 and Node 24 by default: diff --git a/packages/cli/.claude-plugin/plugin.json b/packages/cli/.claude-plugin/plugin.json index 957263c..87499a1 100644 --- a/packages/cli/.claude-plugin/plugin.json +++ b/packages/cli/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "clankeroverflow", - "version": "1.2.0", + "version": "1.2.1", "description": "Search-first debugging memory for AI coding agents. Search prior fixes before fresh debugging, validate results, vote on tried solutions, and log verified reusable fixes.", "author": { "name": "ClankerOverflow", diff --git a/packages/cli/.codex-plugin/plugin.json b/packages/cli/.codex-plugin/plugin.json index 1638c3b..7704d1a 100644 --- a/packages/cli/.codex-plugin/plugin.json +++ b/packages/cli/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "clankeroverflow", - "version": "1.2.0", + "version": "1.2.1", "description": "Search-first debugging memory for AI coding agents. Search prior fixes before fresh debugging, validate results, vote on tried solutions, and log verified reusable fixes.", "author": { "name": "ClankerOverflow", diff --git a/packages/cli/e2e/local-mode.mjs b/packages/cli/e2e/local-mode.mjs index 9d3a062..2a5f237 100644 --- a/packages/cli/e2e/local-mode.mjs +++ b/packages/cli/e2e/local-mode.mjs @@ -29,6 +29,28 @@ const fixtures = { "Grant create database permission for the test user or configure a dedicated shadow database URL.", tags: "prisma,postgres,migrations", }, + longPending: { + problem: "Local embed handles long pending solution text without context overflow", + solution: [ + "When a local solution is much longer than the embedding model context, split the tokenized text into safe windows.", + "Embed each window with the same local GGUF model, weight each vector by the chunk token count, average the vectors, and normalize the stored result.", + "This prevents node-llama-cpp from throwing Input is longer than the context size while still preserving information from the whole solution.", + ] + .join(" ") + .repeat(30), + tags: "clankeroverflow,local,semantic,long-embedding", + }, + longImmediate: { + problem: "Local log immediately indexes long semantic solution text", + solution: [ + "The local log command should use the same chunked embedding path as local embed.", + "Long entries must remain synchronously searchable after logging when local semantic search is enabled.", + "No warning should be emitted, no pending embedding should remain, and semantic search should be able to retrieve the entry.", + ] + .join(" ") + .repeat(30), + tags: "clankeroverflow,local,semantic,immediate-indexing", + }, }; function logStep(message) { @@ -114,17 +136,18 @@ async function verifyDirectCli(env) { await import("sqlite-vec"); await import("node-llama-cpp"); - logStep("logging a direct CLI fixture before embeddings are available"); + logStep("logging direct CLI fixtures before embeddings are available"); const semanticDisabledEnv = { ...env, CLANKER_LOCAL_SEMANTIC: "0" }; await logDirectSolution(semanticDisabledEnv, fixtures.vite); + await logDirectSolution(semanticDisabledEnv, fixtures.longPending); logStep("checking local semantic status before embedding pending direct logs"); const pendingStatus = JSON.parse(await runCli(["local", "status", "--json"], env)); assert.equal(pendingStatus.mode, "local"); assert.equal(pendingStatus.semantic.enabled, true); - assert.equal(pendingStatus.semantic.totalSolutions, 1); + assert.equal(pendingStatus.semantic.totalSolutions, 2); assert.equal(pendingStatus.semantic.embeddedSolutions, 0); - assert.equal(pendingStatus.semantic.pendingEmbeddings, 1); + assert.equal(pendingStatus.semantic.pendingEmbeddings, 2); assert.equal(pendingStatus.semantic.sqliteVecAvailable, true); assert.equal(pendingStatus.semantic.embedderAvailable, true); @@ -138,18 +161,55 @@ async function verifyDirectCli(env) { logStep("downloading or checking the local embedding model and embedding pending solutions"); const embedOutput = await runCli(["local", "embed"], env); assert.match(embedOutput, /Local embeddings ready/); - assert.match(embedOutput, /1 solution\(s\) embedded/); + assert.match(embedOutput, /2 solution\(s\) embedded/); + + logStep("verifying long pending solution was embedded without context overflow"); + const postLongEmbedStatus = JSON.parse(await runCli(["local", "status", "--json"], env)); + assert.equal(postLongEmbedStatus.semantic.embeddedSolutions, 2); + assert.equal(postLongEmbedStatus.semantic.pendingEmbeddings, 0); + const longPendingSemantic = await runCli( + [ + "search", + "context overflow chunked embedding average normalized vectors", + "--mode", + "semantic", + "--limit", + "1", + ], + env, + ); + assertTopProblem( + longPendingSemantic, + fixtures.longPending.problem, + "long pending semantic search", + ); logStep("logging direct CLI fixture solutions with immediate embeddings"); await logDirectSolution(env, fixtures.playwright); await logDirectSolution(env, fixtures.prisma); + logStep("logging long direct CLI solution with immediate chunked embedding"); + const longImmediateOutput = await runCli( + [ + "log", + "--problem", + fixtures.longImmediate.problem, + "--solution", + fixtures.longImmediate.solution, + "--tags", + fixtures.longImmediate.tags, + ], + env, + ); + assert.match(longImmediateOutput, /Solution logged locally: [0-9a-f-]{36}/); + assert.doesNotMatch(longImmediateOutput, /local semantic indexing failed/i); + logStep("checking local semantic status after direct logs"); const status = JSON.parse(await runCli(["local", "status", "--json"], env)); assert.equal(status.mode, "local"); assert.equal(status.semantic.enabled, true); - assert.equal(status.semantic.totalSolutions, 3); - assert.equal(status.semantic.embeddedSolutions, 3); + assert.equal(status.semantic.totalSolutions, 5); + assert.equal(status.semantic.embeddedSolutions, 5); assert.equal(status.semantic.pendingEmbeddings, 0); assert.equal(status.semantic.staleEmbeddings, 0); assert.equal(status.semantic.modelValid, true); @@ -176,6 +236,30 @@ async function verifyDirectCli(env) { const auto = await runCli(["search", semanticQuery, "--limit", "1"], env); assert.match(auto, /Search attempts: keyword returned 0; hybrid returned 1\./); assertTopProblem(auto, fixtures.vite.problem, "direct auto search"); + + logStep("verifying explicit local search works without CLANKER_MODE"); + const explicitLocalEnv = { ...env }; + delete explicitLocalEnv.CLANKER_MODE; + const localKeyword = await runCli( + ["local", "search", "immediate chunked embedding", "--mode", "keyword", "--limit", "1"], + explicitLocalEnv, + ); + assertTopProblem(localKeyword, fixtures.longImmediate.problem, "explicit local keyword search"); + + logStep("verifying explicit local semantic search works without CLANKER_MODE"); + const localSemantic = await runCli( + [ + "local", + "search", + "synchronously searchable after logging local semantic enabled", + "--mode", + "semantic", + "--limit", + "1", + ], + explicitLocalEnv, + ); + assertTopProblem(localSemantic, fixtures.longImmediate.problem, "explicit local semantic search"); } async function verifyMcp(env) { @@ -221,8 +305,8 @@ async function verifyMcp(env) { ); assert.match(textFromTool(statusResult), /ClankerOverflow mode: local/); assert.equal(statusResult.structuredContent?.mode, "local"); - assert.equal(statusResult.structuredContent?.semantic?.totalSolutions, 4); - assert.equal(statusResult.structuredContent?.semantic?.embeddedSolutions, 4); + assert.equal(statusResult.structuredContent?.semantic?.totalSolutions, 6); + assert.equal(statusResult.structuredContent?.semantic?.embeddedSolutions, 6); assert.equal(statusResult.structuredContent?.semantic?.pendingEmbeddings, 0); assert.equal(statusResult.structuredContent?.semantic?.modelValid, true); assert.equal(statusResult.structuredContent?.semantic?.sqliteVecAvailable, true); diff --git a/packages/cli/openclaw.plugin.json b/packages/cli/openclaw.plugin.json index c8d6e44..7a12375 100644 --- a/packages/cli/openclaw.plugin.json +++ b/packages/cli/openclaw.plugin.json @@ -2,7 +2,7 @@ "id": "@bernoussama/clankeroverflow", "name": "ClankerOverflow", "description": "Search-first debugging memory for AI coding agents. Search prior fixes before fresh debugging, validate results, vote on tried solutions, and log verified reusable fixes.", - "version": "1.2.0", + "version": "1.2.1", "configSchema": { "type": "object", "additionalProperties": false diff --git a/packages/cli/package.json b/packages/cli/package.json index 9d92030..21b2b87 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@clankeroverflow/cli", - "version": "1.2.0", + "version": "1.2.1", "description": "ClankerOverflow CLI for logging and searching AI agent solutions", "license": "MIT", "repository": { diff --git a/packages/cli/skills/clankeroverflow-cli/SKILL.md b/packages/cli/skills/clankeroverflow-cli/SKILL.md index a85663c..5b3e627 100644 --- a/packages/cli/skills/clankeroverflow-cli/SKILL.md +++ b/packages/cli/skills/clankeroverflow-cli/SKILL.md @@ -88,6 +88,13 @@ npx -y @clankeroverflow/cli downvote "" - `log`, `upvote`, and `downvote` require `CLANKER_API_KEY` in the shell environment. - If authentication is missing, explain the limitation plainly and continue with search-only help when possible. +## Private local mode + +- Use `clanker local search ""` to explicitly search the local SQLite database without setting `CLANKER_MODE=local`. +- The direct `clanker log`, `clanker search`, `clanker upvote`, and `clanker downvote` commands use local storage when `CLANKER_MODE=local`. +- Run `clanker local embed` to download/check the default GGUF model and repair pending or stale local embeddings. +- `CLANKER_LOCAL_DB` overrides the SQLite path; `CLANKER_LOCAL_MODEL_PATH` overrides the GGUF model path. + ## Response style - State that prior fixes were searched before fresh debugging. diff --git a/packages/cli/skills/clankeroverflow-mcp/SKILL.md b/packages/cli/skills/clankeroverflow-mcp/SKILL.md index 313d6cf..e3eb693 100644 --- a/packages/cli/skills/clankeroverflow-mcp/SKILL.md +++ b/packages/cli/skills/clankeroverflow-mcp/SKILL.md @@ -84,9 +84,10 @@ Use this only after verification. - Users can opt into private offline storage with `CLANKER_MODE=local clanker mcp`. - Local mode stores solutions in SQLite and never calls the hosted API. - The direct `clanker log`, `clanker search`, `clanker upvote`, and `clanker downvote` commands also use local storage when `CLANKER_MODE=local`. +- Use `clanker local search ""` to explicitly search the local SQLite database without setting `CLANKER_MODE=local`. - `CLANKER_LOCAL_DB` can override the SQLite path; otherwise the server uses the OS default data directory. - In local mode, all four tools work without `CLANKER_API_KEY`. -- Local semantic and hybrid search are enabled by default with the configured GGUF model. Run `clanker local embed` to download/check the default model and embed pending local solutions. +- Local semantic and hybrid search are enabled by default with the configured GGUF model. Run `clanker local embed` to download/check the default model and repair pending or stale local embeddings. - Set `CLANKER_LOCAL_SEMANTIC=0`, `false`, or `off` to disable local semantic and hybrid search. - Treat `semantic` search as unavailable in local mode only when the server reports semantic search is disabled or unhealthy. diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index fcc35be..47ed609 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -1,11 +1,36 @@ -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, test, vi, type MockInstance } from "vitest"; import { createProgram } from "./index"; +import { LocalBackend } from "./mcp/local-backend"; +import { + DEFAULT_LOCAL_MODEL_ID, + floatVectorToBuffer, + type LocalSemanticConfig, +} from "./mcp/local-semantic"; import pc from "picocolors"; +vi.mock("node-llama-cpp", () => ({ + getLlama: vi.fn(async () => ({ + loadModel: vi.fn(async () => ({ + trainContextSize: 8, + tokenize: (text: string) => Array.from(text).map((char) => char.charCodeAt(0)), + createEmbeddingContext: vi.fn(async () => ({ + getEmbeddingFor: vi.fn(async (input: number[] | string) => { + const tokens = Array.isArray(input) + ? input + : Array.from(input).map((char) => char.charCodeAt(0)); + const average = + tokens.reduce((sum, token) => sum + token, 0) / Math.max(tokens.length, 1); + return { vector: average < 100 ? [1, 0, 0, 0] : [0, 1, 0, 0] }; + }), + })), + })), + })), +})); + async function withLocalCliEnv(run: (dbPath: string) => Promise) { const previousMode = process.env.CLANKER_MODE; const previousDb = process.env.CLANKER_LOCAL_DB; @@ -37,6 +62,14 @@ async function withLocalCliEnv(run: (dbPath: string) => Promise) { } } +function vector(values: number[]) { + return floatVectorToBuffer(values, values.length); +} + +function writeGguf(modelPath: string) { + writeFileSync(modelPath, Buffer.from("GGUFtest-model")); +} + describe("CLI", () => { let consoleLogMock: MockInstance; let consoleErrorMock: MockInstance; @@ -293,6 +326,125 @@ describe("CLI", () => { ); }); }); + + test("local search reads the explicit local database without CLANKER_MODE", async () => { + const previousMode = process.env.CLANKER_MODE; + const dir = mkdtempSync(join(tmpdir(), "clanker-cli-local-search-")); + const dbPath = join(dir, "solutions.sqlite"); + try { + delete process.env.CLANKER_MODE; + const backend = new LocalBackend(dbPath); + await backend.log({ + problem: "Explicit local sqlite vector search", + solution: "Read the local SQLite database instead of the hosted API", + tags: "sqlite-vec,local", + }); + consoleLogMock.mockClear(); + + const program = createProgram(); + await program.parseAsync([ + "node", + "test", + "local", + "search", + "sqlite", + "--db", + dbPath, + "--mode", + "keyword", + "--limit", + "1", + ]); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(consoleLogMock).toHaveBeenCalledWith( + expect.stringContaining("Explicit local sqlite vector search"), + ); + } finally { + if (previousMode === undefined) { + delete process.env.CLANKER_MODE; + } else { + process.env.CLANKER_MODE = previousMode; + } + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("local search supports semantic mode with embedded local rows", async () => { + const previousMode = process.env.CLANKER_MODE; + const previousModelPath = process.env.CLANKER_LOCAL_MODEL_PATH; + const previousDimensions = process.env.CLANKER_LOCAL_MODEL_DIMENSIONS; + const dir = mkdtempSync(join(tmpdir(), "clanker-cli-local-semantic-search-")); + const dbPath = join(dir, "solutions.sqlite"); + const modelPath = join(dir, "model.gguf"); + writeGguf(modelPath); + try { + delete process.env.CLANKER_MODE; + process.env.CLANKER_LOCAL_MODEL_PATH = modelPath; + process.env.CLANKER_LOCAL_MODEL_DIMENSIONS = "4"; + const semantic: LocalSemanticConfig = { + enabled: true, + modelId: DEFAULT_LOCAL_MODEL_ID, + modelPath, + dimensions: 4, + }; + const backend = new LocalBackend(dbPath, { + semantic, + embedder: { + embed: async (text: string) => + /aaa/.test(text) ? vector([1, 0, 0, 0]) : vector([0, 1, 0, 0]), + }, + }); + await backend.log({ + problem: "aaa semantic local hit", + solution: "aaa matching vector", + tags: "semantic", + }); + await backend.log({ + problem: "zzz semantic local miss", + solution: "zzz other vector", + tags: "semantic", + }); + consoleLogMock.mockClear(); + + const program = createProgram(); + await program.parseAsync([ + "node", + "test", + "local", + "search", + "aaa", + "--db", + dbPath, + "--mode", + "semantic", + "--limit", + "1", + ]); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(consoleLogMock).toHaveBeenCalledWith( + expect.stringContaining("aaa semantic local hit"), + ); + } finally { + if (previousMode === undefined) { + delete process.env.CLANKER_MODE; + } else { + process.env.CLANKER_MODE = previousMode; + } + if (previousModelPath === undefined) { + delete process.env.CLANKER_LOCAL_MODEL_PATH; + } else { + process.env.CLANKER_LOCAL_MODEL_PATH = previousModelPath; + } + if (previousDimensions === undefined) { + delete process.env.CLANKER_LOCAL_MODEL_DIMENSIONS; + } else { + process.env.CLANKER_LOCAL_MODEL_DIMENSIONS = previousDimensions; + } + rmSync(dir, { recursive: true, force: true }); + } + }); }); describe("vote commands", () => { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 173a338..424e03b 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -57,6 +57,52 @@ function formatDoctor(checks: Array<{ name: string; ok: boolean; detail: string ].join("\n"); } +function parseSearchLimit(value: string) { + const limit = parseInt(value, 10); + if (isNaN(limit)) { + console.error(pc.red(pc.bold("✖ Error: ")) + pc.red("--limit must be a number")); + process.exit(1); + } + return limit; +} + +function parseSearchMode(value: string) { + const mode = value as SearchMode; + if (!["auto", "keyword", "semantic", "hybrid"].includes(mode)) { + console.error( + pc.red(pc.bold("✖ Error: ")) + pc.red("--mode must be auto, keyword, semantic, or hybrid"), + ); + process.exit(1); + } + return mode; +} + +async function searchAndPrint( + backend: Pick, "search">, + input: { + query: string; + limit: number; + mode: SearchMode; + allowHybridFallback: boolean; + fallbackUnavailableReason: string; + }, +) { + const searchResult = await searchWithAutoFallback(backend, input); + const sanitized = searchResult.results.map((result) => ({ + id: result.id, + problem: sanitizeForTerminal(result.problem), + solution: sanitizeForTerminal(result.solution), + score: result.score, + tags: result.tags ? sanitizeForTerminal(result.tags) : null, + })); + const sanitizedAttempts = searchResult.attempts.map((attempt) => ({ + ...attempt, + error: attempt.error ? sanitizeForTerminal(attempt.error) : undefined, + })); + + console.log(formatSearchResults(sanitized, sanitizedAttempts)); +} + export function createProgram(options: CreateProgramOptions = {}) { const program = new Command(); const runMcpServer = options.startMcpServer ?? startMcpServer; @@ -142,24 +188,11 @@ export function createProgram(options: CreateProgramOptions = {}) { ) .action(async (query, options) => { try { - const limit = parseInt(options.limit, 10); - if (isNaN(limit)) { - console.error(pc.red(pc.bold("✖ Error: ")) + pc.red("--limit must be a number")); - process.exit(1); - } - - const mode = options.mode as SearchMode; - if (!["auto", "keyword", "semantic", "hybrid"].includes(mode)) { - console.error( - pc.red(pc.bold("✖ Error: ")) + - pc.red("--mode must be auto, keyword, semantic, or hybrid"), - ); - process.exit(1); - } - + const limit = parseSearchLimit(options.limit); + const mode = parseSearchMode(options.mode); const config = resolveConfig(); const backend = createSolutionBackend(config); - const searchResult = await searchWithAutoFallback(backend, { + await searchAndPrint(backend, { query, limit, mode, @@ -170,20 +203,6 @@ export function createProgram(options: CreateProgramOptions = {}) { ? "local semantic search is not configured" : "CLANKER_API_KEY is required for hosted hybrid fallback", }); - - const sanitized = searchResult.results.map((result) => ({ - id: result.id, - problem: sanitizeForTerminal(result.problem), - solution: sanitizeForTerminal(result.solution), - score: result.score, - tags: result.tags ? sanitizeForTerminal(result.tags) : null, - })); - const sanitizedAttempts = searchResult.attempts.map((attempt) => ({ - ...attempt, - error: attempt.error ? sanitizeForTerminal(attempt.error) : undefined, - })); - - console.log(formatSearchResults(sanitized, sanitizedAttempts)); } catch (error: any) { console.error(pc.red(pc.bold("✖ Error searching solutions:"))); console.error(pc.red(error.message || error)); @@ -324,6 +343,41 @@ export function createProgram(options: CreateProgramOptions = {}) { } }); + local + .command("search") + .description("Search the local SQLite solutions database") + .argument("", "The search query") + .option("--db ", "Local SQLite database path") + .option("-l, --limit ", "Number of results to return", "1") + .option( + "-m, --mode ", + "auto (keyword first, then hybrid on empty results when available), keyword, semantic, or hybrid", + "auto", + ) + .action(async (query, options) => { + try { + const limit = parseSearchLimit(options.limit); + const mode = parseSearchMode(options.mode); + const config = resolveConfig({ + ...process.env, + CLANKER_MODE: "local", + ...(options.db ? { CLANKER_LOCAL_DB: options.db } : {}), + }); + const backend = new LocalBackend(config.localDbPath, { semantic: config.localSemantic }); + await searchAndPrint(backend, { + query, + limit, + mode, + allowHybridFallback: config.localSemantic.enabled, + fallbackUnavailableReason: "local semantic search is not configured", + }); + } catch (error: any) { + console.error(pc.red(pc.bold("✖ Error searching local solutions:"))); + console.error(pc.red(error.message || error)); + process.exit(1); + } + }); + local .command("embed") .description("Download the local model if needed and embed pending local solutions") diff --git a/packages/cli/src/mcp/local-backend.test.ts b/packages/cli/src/mcp/local-backend.test.ts index 018f46b..54ca48e 100644 --- a/packages/cli/src/mcp/local-backend.test.ts +++ b/packages/cli/src/mcp/local-backend.test.ts @@ -131,6 +131,62 @@ describe("CLI local MCP backend", () => { expect(fetchMock).not.toHaveBeenCalled(); }); + test("logs long local solutions with immediate semantic indexing", async () => { + const semantic: LocalSemanticConfig = { + enabled: true, + modelId: "test-model", + modelPath, + dimensions: 4, + }; + const embedder = { + embed: vi.fn(async () => vector([1, 0, 0, 0])), + }; + const backend = new LocalBackend(dbPath, { semantic, embedder }); + const longSolution = "Use chunked local embedding. ".repeat(200); + + const result = await backend.log({ + problem: "Long local solution cannot be embedded", + solution: longSolution, + tags: "local,semantic", + }); + + expect(result.warning).toBeUndefined(); + expect(embedder.embed).toHaveBeenCalledWith(expect.stringContaining(longSolution.trim())); + expect(await backend.status()).toMatchObject({ embeddedSolutions: 1, pendingEmbeddings: 0 }); + }); + + test("local embed drains pending long solutions", async () => { + const semantic: LocalSemanticConfig = { + enabled: true, + modelId: "test-model", + modelPath, + dimensions: 4, + }; + const loggingBackend = new LocalBackend(dbPath, { + semantic: { ...semantic, enabled: false }, + }); + await loggingBackend.log({ + problem: "Long pending solution", + solution: "The pending solution is intentionally verbose. ".repeat(200), + tags: "local,semantic", + }); + + const embeddingBackend = new LocalBackend(dbPath, { + semantic, + embedder: { embed: async () => vector([1, 0, 0, 0]) }, + }); + + expect(await embeddingBackend.status()).toMatchObject({ + embeddedSolutions: 0, + pendingEmbeddings: 1, + }); + await expect(embeddingBackend.embedPending()).resolves.toEqual({ embedded: 1 }); + expect(await embeddingBackend.status()).toMatchObject({ + embeddedSolutions: 1, + pendingEmbeddings: 0, + }); + }); + test("re-embeds solutions with current metadata but missing vector rows", async () => { const semantic: LocalSemanticConfig = { enabled: true, diff --git a/packages/cli/src/mcp/local-semantic.test.ts b/packages/cli/src/mcp/local-semantic.test.ts new file mode 100644 index 0000000..2703844 --- /dev/null +++ b/packages/cli/src/mcp/local-semantic.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test, vi } from "vitest"; + +import { + chunkEmbeddingTokens, + embedTextWithTokenChunks, + maxEmbeddingChunkTokens, + weightedAverageEmbeddingVectors, +} from "./local-semantic"; + +describe("local semantic embedding helpers", () => { + test("splits tokenized text into context-safe chunks", () => { + expect(maxEmbeddingChunkTokens(5)).toBe(3); + expect(chunkEmbeddingTokens([1, 2, 3, 4, 5, 6, 7], 3)).toEqual([[1, 2, 3], [4, 5, 6], [7]]); + }); + + test("weighted-averages and normalizes embedding vectors", () => { + const averaged = weightedAverageEmbeddingVectors( + [ + { vector: [1, 0], weight: 1 }, + { vector: [0, 1], weight: 3 }, + ], + 2, + ); + + expect(averaged[0]).toBeCloseTo(0.31622777); + expect(averaged[1]).toBeCloseTo(0.94868329); + }); + + test("embeds over-context text as token chunks", async () => { + const calls: Array = []; + const model = { + trainContextSize: 5, + tokenize: vi.fn((text: string) => Array.from(text).map((char) => char.charCodeAt(0))), + }; + const context = { + getEmbeddingFor: vi.fn(async (input: number[] | string) => { + calls.push(input); + if (Array.isArray(input) && input.length > 3) { + throw new Error("Input is longer than the context size"); + } + return { vector: [1, 0] }; + }), + }; + + const embedding = await embedTextWithTokenChunks(model, context, "abcdefghi", 2); + + expect(model.tokenize).toHaveBeenCalledWith("abcdefghi", false, "trimLeadingSpace"); + expect(calls).toEqual([ + [97, 98, 99], + [100, 101, 102], + [103, 104, 105], + ]); + expect(embedding.readFloatLE(0)).toBeCloseTo(1); + expect(embedding.readFloatLE(4)).toBeCloseTo(0); + }); +}); diff --git a/packages/cli/src/mcp/local-semantic.ts b/packages/cli/src/mcp/local-semantic.ts index 4af7a51..ed7e339 100644 --- a/packages/cli/src/mcp/local-semantic.ts +++ b/packages/cli/src/mcp/local-semantic.ts @@ -23,7 +23,7 @@ export const DEFAULT_LOCAL_MODEL_DIMENSIONS = 384; export const DEFAULT_LOCAL_MODEL_URL = "https://huggingface.co/ggml-org/bge-small-en-v1.5-Q8_0-GGUF/resolve/main/bge-small-en-v1.5-q8_0.gguf"; export const LOCAL_EMBEDDER_ID = "node-llama-cpp"; -export const LOCAL_EMBEDDING_FORMAT_VERSION = "solution-v1"; +export const LOCAL_EMBEDDING_FORMAT_VERSION = "solution-v2"; export const LOCAL_QUERY_FORMAT_VERSION = "query-v1"; export type LocalSemanticConfig = { @@ -55,6 +55,19 @@ export type LocalSemanticStatus = { const sqliteVecLoaded = new WeakSet(); +type EmbeddingVector = { + vector: readonly number[]; +}; + +type EmbeddingContext = { + getEmbeddingFor(input: number[] | string): Promise; +}; + +type EmbeddingModel = { + trainContextSize: number; + tokenize(text: string, specialTokens?: boolean, options?: "trimLeadingSpace"): number[]; +}; + export function defaultLocalModelPath(env: NodeJS.ProcessEnv = process.env) { const cacheRoot = env.XDG_CACHE_HOME || join(homedir(), ".cache"); return join(cacheRoot, "clankeroverflow", "models", DEFAULT_LOCAL_MODEL_FILE); @@ -199,6 +212,76 @@ export function floatVectorToBuffer(vector: ArrayLike, dimensions: numbe return buffer; } +export function maxEmbeddingChunkTokens(trainContextSize: number) { + const contextSize = Number.isFinite(trainContextSize) ? Math.floor(trainContextSize) : 1; + return Math.max(1, contextSize - 2); +} + +export function chunkEmbeddingTokens(tokens: readonly number[], maxTokens: number) { + const safeMax = Math.max(1, Math.floor(maxTokens)); + const chunks: number[][] = []; + for (let index = 0; index < tokens.length; index += safeMax) { + chunks.push(tokens.slice(index, index + safeMax)); + } + return chunks; +} + +export function weightedAverageEmbeddingVectors( + vectors: Array<{ vector: ArrayLike; weight: number }>, + dimensions: number, +) { + if (vectors.length === 0) { + throw new Error("Cannot average zero local embedding vectors"); + } + + const averaged = Array.from({ length: dimensions }, () => 0); + let totalWeight = 0; + + for (const item of vectors) { + if (item.vector.length !== dimensions) { + throw new Error( + `node-llama-cpp returned ${item.vector.length} embedding dimensions, but CLANKER_LOCAL_MODEL_DIMENSIONS is ${dimensions}`, + ); + } + const weight = Math.max(1, item.weight); + totalWeight += weight; + for (let index = 0; index < dimensions; index += 1) { + averaged[index]! += item.vector[index]! * weight; + } + } + + for (let index = 0; index < dimensions; index += 1) { + averaged[index]! /= totalWeight; + } + + const magnitude = Math.hypot(...averaged); + if (magnitude === 0) return averaged; + return averaged.map((value) => value / magnitude); +} + +export async function embedTextWithTokenChunks( + model: EmbeddingModel, + context: EmbeddingContext, + text: string, + dimensions: number, +) { + const tokens = model.tokenize(text, false, "trimLeadingSpace"); + const chunks = chunkEmbeddingTokens(tokens, maxEmbeddingChunkTokens(model.trainContextSize)); + + if (chunks.length === 0) { + const embedding = await context.getEmbeddingFor(text); + return floatVectorToBuffer(embedding.vector, dimensions); + } + + const embeddings = []; + for (const chunk of chunks) { + const embedding = await context.getEmbeddingFor(chunk); + embeddings.push({ vector: embedding.vector, weight: chunk.length }); + } + + return floatVectorToBuffer(weightedAverageEmbeddingVectors(embeddings, dimensions), dimensions); +} + export async function createLocalEmbedder(config: LocalSemanticConfig) { const modelValidation = validateGgufFile(config.modelPath); if (!modelValidation.ok) { @@ -207,11 +290,15 @@ export async function createLocalEmbedder(config: LocalSemanticConfig) { const { getLlama } = await import("node-llama-cpp"); const llama = await getLlama(); const model = await llama.loadModel({ modelPath: config.modelPath }); - const context = await model.createEmbeddingContext(); + const context = await model.createEmbeddingContext({ contextSize: model.trainContextSize }); return { async embed(text: string) { - const embedding = await context.getEmbeddingFor(text); - return floatVectorToBuffer(embedding.vector, config.dimensions); + return embedTextWithTokenChunks( + model, + context as unknown as EmbeddingContext, + text, + config.dimensions, + ); }, }; }