From ea0cbfa0259aeec9d4fa06feb67b5418fdc5f4a6 Mon Sep 17 00:00:00 2001 From: Oussama Bernou Date: Mon, 22 Jun 2026 12:18:41 +0100 Subject: [PATCH] fix: wire advanced FTS5 parser into local keyword search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operator-aware ftsQuery parser (AND/OR/NOT, column filters, prefix terms, NEAR, syntax validation) was only exercised by unit tests; the runtime keyword path used the simpler localFtsQuery, so documented advanced syntax was silently ignored and syntax errors never surfaced. - local-backend: route searchLocalKeywordExact through ftsQuery so advanced operators, column filters, and syntax validation work as documented; delete the now-dead localFtsQuery - local-backend: fix NEAR passthrough — tokenizeFtsQuery now captures NEAR(...) as a unit and normalizes the forgiving comma form (NEAR(a, b, 5)) to FTS5's space-separated form (NEAR(a b, 5)) - local-backend: treat a leading hyphen as punctuation in simple mode (e.g. "sqlite -wal" searches for both terms) instead of rejecting it - local-backend: correct the FTS5 syntax hint (NEAR uses space-separated terms, not commas) - server: surface FtsQuerySyntaxError as text content instead of re-throwing, so MCP clients can read the hint and retry - index: special-case FtsQuerySyntaxError in the CLI catch blocks with a clearer "Invalid search syntax" header - search-solutions.md: show the canonical NEAR(token nft, 5) form --- packages/cli/commands/search-solutions.md | 2 +- packages/cli/src/index.ts | 14 ++- packages/cli/src/mcp/local-backend.test.ts | 85 ++++++++++++++++- packages/cli/src/mcp/local-backend.ts | 101 ++++++++++++++------- packages/cli/src/mcp/server.ts | 14 ++- 5 files changed, 175 insertions(+), 41 deletions(-) diff --git a/packages/cli/commands/search-solutions.md b/packages/cli/commands/search-solutions.md index ff3b6ed..6f04a7f 100644 --- a/packages/cli/commands/search-solutions.md +++ b/packages/cli/commands/search-solutions.md @@ -11,7 +11,7 @@ Search ClankerOverflow for solutions matching the query. Use this as the first s Keep keyword queries short. Start with the smallest distinctive literal fingerprint: an error code, command, package, or short sanitized error phrase. Use tags as relevance signals. Add one package, command, or tag only if the first search is too broad. -**Advanced keyword syntax (local FTS5)**: in keyword/hybrid/auto mode, a query may use FTS5 operators when it contains them, e.g. `database AND crash`, `"oauth callback" OR react*`, `tags:react hooks`, `database NOT physics`, `(a OR b) AND c`, or `NEAR(token, nft, 5)`. Unknown columns, unbalanced parentheses, doubled operators, or stray operators are rejected with a clear message. To search for operator words literally (e.g. the literal text `AND`), wrap the whole query in double quotes. +**Advanced keyword syntax (local FTS5)**: in keyword/hybrid/auto mode, a query may use FTS5 operators when it contains them, e.g. `database AND crash`, `"oauth callback" OR react*`, `tags:react hooks`, `database NOT physics`, `(a OR b) AND c`, or `NEAR(token nft, 5)`. Unknown columns, unbalanced parentheses, doubled operators, or stray operators are rejected with a clear message. To search for operator words literally (e.g. the literal text `AND`), wrap the whole query in double quotes. **Negative/leading-dash values**: to search for a query that itself starts with `-` (a negative number, a version string like `v2.0-beta-1`), separate options from the query with `--`, e.g. `clanker search -- -1`. diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index d653ddd..7919d53 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -18,7 +18,7 @@ import { import { createSolutionBackend } from "./mcp/create-backend.js"; import { startMcpServer } from "./mcp/server.js"; import { formatSearchResults } from "./mcp/format.js"; -import { LocalBackend } from "./mcp/local-backend.js"; +import { FtsQuerySyntaxError, LocalBackend } from "./mcp/local-backend.js"; import { downloadDefaultLocalModel } from "./mcp/local-semantic.js"; import { hasSetupFailures, setupAgents, type Agent, type SkillSelection } from "./setup.js"; import pc from "picocolors"; @@ -322,7 +322,11 @@ export function createProgram(options: CreateProgramOptions = {}) { source: backendMode, }); } catch (error: any) { - console.error(pc.red(pc.bold("✖ Error searching solutions:"))); + if (error instanceof FtsQuerySyntaxError) { + console.error(pc.red(pc.bold("✖ Invalid search syntax:"))); + } else { + console.error(pc.red(pc.bold("✖ Error searching solutions:"))); + } console.error(pc.red(error.message || error)); process.exit(1); } @@ -567,7 +571,11 @@ export function createProgram(options: CreateProgramOptions = {}) { source: "local", }); } catch (error: any) { - console.error(pc.red(pc.bold("✖ Error searching local solutions:"))); + if (error instanceof FtsQuerySyntaxError) { + console.error(pc.red(pc.bold("✖ Invalid search syntax:"))); + } else { + console.error(pc.red(pc.bold("✖ Error searching local solutions:"))); + } console.error(pc.red(error.message || error)); process.exit(1); } diff --git a/packages/cli/src/mcp/local-backend.test.ts b/packages/cli/src/mcp/local-backend.test.ts index b5af8e7..732247d 100644 --- a/packages/cli/src/mcp/local-backend.test.ts +++ b/packages/cli/src/mcp/local-backend.test.ts @@ -374,6 +374,70 @@ describe("CLI local MCP backend", () => { "search query must not be empty", ); }); + + test("exact keyword search honors the AND operator", async () => { + const backend = new LocalBackend(dbPath); + const { id } = await backend.log({ + problem: "OAuth callback timeout", + solution: "Keep waitUntil tasks alive", + tags: "auth", + }); + await backend.log({ + problem: "OAuth misconfiguration only", + solution: "Check redirect URIs.", + tags: "auth", + }); + + const results = await backend.search({ + query: "OAuth AND timeout", + limit: 5, + mode: "keyword", + keywordStrategy: "exact", + }); + expect(results).toHaveLength(1); + expect(results[0]!.id).toBe(id); + }); + + test("exact keyword search honors column filters", async () => { + const backend = new LocalBackend(dbPath); + await backend.log({ + problem: "OAuth callback timeout", + solution: "Keep waitUntil tasks alive", + tags: "auth", + }); + await backend.log({ + problem: "Startup race condition", + solution: "Await initialization.", + tags: "init", + }); + + const results = await backend.search({ + query: "tags:auth", + limit: 5, + mode: "keyword", + keywordStrategy: "exact", + }); + expect(results).toHaveLength(1); + expect(results[0]!.problem).toBe("OAuth callback timeout"); + }); + + test("exact keyword search rejects a malformed advanced query", async () => { + const backend = new LocalBackend(dbPath); + await backend.log({ + problem: "Database crash", + solution: "Restart the service.", + tags: "db", + }); + + await expect( + backend.search({ + query: "database AND", + limit: 5, + mode: "keyword", + keywordStrategy: "exact", + }), + ).rejects.toThrow(FtsQuerySyntaxError); + }); }); describe("ftsQuery", () => { @@ -396,8 +460,9 @@ describe("ftsQuery", () => { expect(ftsQuery(" ")).toBe(""); }); - test("simple mode rejects bare leading-dash negation", () => { - expect(() => ftsQuery("-foo")).toThrow(FtsQuerySyntaxError); + test("simple mode treats a leading dash as punctuation, not negation", () => { + expect(ftsQuery("-foo")).toBe('"foo"'); + expect(ftsQuery("sqlite -wal")).toBe('"sqlite" "wal"'); }); test("advanced mode preserves the AND operator", () => { @@ -416,6 +481,22 @@ describe("ftsQuery", () => { expect(ftsQuery("tags:react hooks")).toBe('tags : "react" AND "hooks"'); }); + test("advanced mode preserves balanced parentheses as a group", () => { + expect(ftsQuery("(database OR crash) AND startup")).toBe( + '( "database" OR "crash" ) AND "startup"', + ); + }); + + test("advanced mode normalizes NEAR(...) comma form to space-separated FTS5", () => { + expect(ftsQuery("NEAR(token, nft, 5)")).toBe("NEAR(token nft, 5)"); + expect(ftsQuery("NEAR(oauth timeout)")).toBe("NEAR(oauth timeout)"); + expect(ftsQuery("NEAR(token, nft)")).toBe("NEAR(token nft)"); + }); + + test("advanced mode rejects an unterminated NEAR(...) expression", () => { + expect(() => ftsQuery("NEAR(token nft")).toThrow(FtsQuerySyntaxError); + }); + test("advanced mode rejects unknown column filters", () => { expect(() => ftsQuery("foo:bar")).toThrow(FtsQuerySyntaxError); }); diff --git a/packages/cli/src/mcp/local-backend.ts b/packages/cli/src/mcp/local-backend.ts index 2351f27..5444863 100644 --- a/packages/cli/src/mcp/local-backend.ts +++ b/packages/cli/src/mcp/local-backend.ts @@ -37,29 +37,6 @@ function nowIso() { return new Date().toISOString(); } -export function localFtsQuery(query: string) { - const normalized = query - .replace(/https?:\/\/\S+/gi, (match) => match.replace(/[/:.?=&%#-]+/g, " ")) - .trim(); - const phrases = [...normalized.matchAll(/"([^"]+)"/g)].map((match) => match[1]?.trim() ?? ""); - const withoutPhrases = normalized.replace(/"[^"]+"/g, " "); - const terms = withoutPhrases.match(/[\p{L}\p{N}_][\p{L}\p{N}_./:-]*/gu) ?? []; - const phraseClauses = phrases - .map((phrase) => phrase.replace(/[^\p{L}\p{N}_-]+/gu, " ").trim()) - .filter(Boolean) - .map((phrase) => `"${phrase.replaceAll('"', '""')}"`); - const termClauses = terms - .flatMap((term) => - term - .replace(/[^\p{L}\p{N}_-]+/gu, " ") - .trim() - .split(/\s+/), - ) - .filter(Boolean) - .map((term) => `"${term.replaceAll('"', '""')}"`); - return [...new Set([...phraseClauses, ...termClauses])].join(" "); -} - export class FtsQuerySyntaxError extends Error {} /** FTS5 column names backed by the solution_fts(problem, solution, tags) table. */ @@ -70,7 +47,7 @@ const FTS_BOOLEAN_OPERATORS = new Set(["AND", "OR", "NOT"]); function ftsSyntaxError(message: string): never { throw new FtsQuerySyntaxError( `${message} Valid FTS5 syntax: quoted "phrases", prefix term*, ` + - `column:term, term AND/OR/NOT term, (group), NEAR(term, term, N). ` + + `column:term, term AND/OR/NOT term, (group), NEAR(term term [, N]). ` + `To search for these words literally, wrap the whole query in double quotes.`, ); } @@ -111,7 +88,8 @@ function tokenizeFtsQuery(query: string): FtsToken[] { } }; - for (const char of query) { + for (let i = 0; i < query.length; i += 1) { + const char = query[i]!; if (inQuotes) { current += char; if (char === '"') inQuotes = false; @@ -122,6 +100,17 @@ function tokenizeFtsQuery(query: string): FtsToken[] { inQuotes = true; continue; } + // Capture a whole NEAR(...) expression as one token so the comma-separated + // form (e.g. NEAR(token, nft, 5)) survives as a unit and can be normalized + // to FTS5's space-separated form. Parens inside quotes are already handled + // above by the inQuotes branch. + if (char === "(" && /^NEAR$/i.test(current.trim())) { + const near = captureNear(query, i); + current = ""; + tokens.push({ type: "word", value: near.value }); + i = near.endIndex; + continue; + } if (char === "(" || char === ")") { flushWord(); tokens.push(char === "(" ? { type: "lparen" } : { type: "rparen" }); @@ -138,6 +127,51 @@ function tokenizeFtsQuery(query: string): FtsToken[] { return tokens; } +/** + * Normalize a `NEAR(...)` expression to FTS5's space-separated form. FTS5 wants + * `NEAR(term1 term2 [, N])` where terms are space-separated and the optional + * integer distance follows a comma. The forgiving comma-separated form + * `NEAR(a, b, 5)` is translated by replacing term-separating commas with spaces + * while preserving a trailing `, N` distance argument. + */ +function normalizeNear(raw: string): string { + const match = raw.match(/^NEAR\s*\((.*)\)$/is); + if (!match) ftsSyntaxError("malformed NEAR(...) expression."); + const inner = match[1]!.trim(); + if (!inner) ftsSyntaxError("NEAR(...) needs at least one term."); + // Split on commas, drop empties, so "a, b, 5" -> ["a","b","5"] and "a b, 5" -> ["a b","5"]. + const parts = inner + .split(",") + .map((part) => part.trim()) + .filter(Boolean); + // A trailing bare integer is the NEAR distance; keep it after a comma. + const last = parts[parts.length - 1]; + const distance = parts.length >= 2 && /^\d+$/.test(last!) ? `, ${last}` : ""; + const terms = (distance ? parts.slice(0, -1) : parts).join(" ").trim(); + if (!terms) ftsSyntaxError("NEAR(...) needs at least one term."); + return `NEAR(${terms}${distance})`; +} + +/** + * Capture a `NEAR(...)` expression starting at the `(` located at `parenIndex`, + * returning the raw text (including `NEAR(` ... `)`) and the index of the final + * `)`. Terms inside are left as-is; the caller normalizes commas to spaces. + */ +function captureNear(query: string, parenIndex: number): { value: string; endIndex: number } { + let depth = 0; + let j = parenIndex; + for (; j < query.length; j += 1) { + const char = query[j]!; + if (char === "(") depth += 1; + else if (char === ")") { + depth -= 1; + if (depth === 0) break; + } + } + if (depth !== 0) ftsSyntaxError('unmatched "(" in NEAR(...) expression.'); + return { value: query.slice(parenIndex - 4, j + 1), endIndex: j }; +} + /** Convert a quoted `"..."` token (with surrounding quotes) into a phrase token. */ function phraseToken(raw: string): FtsToken { // Strip the outer quotes; inner content is the literal phrase. @@ -250,7 +284,7 @@ function buildAdvancedQuery(tokens: FtsToken[]) { if (!expectTerm) parts.push("AND"); // implicit AND between adjacent terms parts.push( token.type === "word" && /^NEAR\s*\(/i.test(token.value) - ? token.value + ? normalizeNear(token.value) : renderAdvancedTerm(token), ); expectTerm = false; @@ -282,15 +316,14 @@ function buildSimpleQuery(query: string) { `Drop them or use explicit AND/OR/NOT operators.`, ); } - const terms = withoutPhrases.match(/-?[\p{L}\p{N}_][\p{L}\p{N}_./:-]*/gu) ?? []; + const terms = withoutPhrases.match(/[\p{L}\p{N}_][\p{L}\p{N}_./:-]*/gu) ?? []; const rendered = [...phrases, ...terms] .map((term) => { - if (term.startsWith("-")) { - ftsSyntaxError( - `bare negation "-${term.slice(1)}" needs a positive term first; use FTS5 NOT instead.`, - ); - } - const clean = term.replace(/[^\p{L}\p{N}_-]+/gu, " ").trim(); + // Treat a leading hyphen (e.g. "sqlite -wal") as punctuation, not FTS5 + // NOT: strip it so the term still matches. Bare negation must be spelled + // out explicitly with the FTS5 NOT operator in advanced mode. + const stripped = term.replace(/^-+/, ""); + const clean = stripped.replace(/[^\p{L}\p{N}_-]+/gu, " ").trim(); return clean ? quoteFtsTerm(clean) : ""; }) .filter(Boolean); @@ -387,7 +420,7 @@ function searchLocalKeywordExpression(db: LocalDb, query: string, limit: number) } export function searchLocalKeywordExact(db: LocalDb, queryText: string, limit: number) { - return searchLocalKeywordExpression(db, localFtsQuery(queryText.trim()), limit); + return searchLocalKeywordExpression(db, ftsQuery(queryText.trim()), limit); } export function searchLocalKeywordRelaxed(db: LocalDb, queryText: string, limit: number) { diff --git a/packages/cli/src/mcp/server.ts b/packages/cli/src/mcp/server.ts index 4d16754..643c200 100644 --- a/packages/cli/src/mcp/server.ts +++ b/packages/cli/src/mcp/server.ts @@ -9,7 +9,11 @@ import type { SolutionBackend } from "./backend.js"; import { modeForSource, resolveConfig, type ServerConfig } from "./config.js"; import { createSolutionBackend } from "./create-backend.js"; import { formatSearchResults } from "./format.js"; -import { LocalBackend, LocalSemanticSearchNotConfiguredError } from "./local-backend.js"; +import { + FtsQuerySyntaxError, + LocalBackend, + LocalSemanticSearchNotConfiguredError, +} from "./local-backend.js"; const logger = new McpLogger({ name: packageJson.name }); @@ -157,6 +161,14 @@ export function createMcpServer(config: ServerConfig = resolveConfig()) { }); return { content: [{ type: "text" as const, text: error.message }] }; } + if (error instanceof FtsQuerySyntaxError) { + logger.warn("Invalid FTS5 search syntax", { + error: error.message, + query, + mode, + }); + return { content: [{ type: "text" as const, text: error.message }] }; + } logger.error("search_solutions failed", { error: error instanceof Error ? error.message : String(error), query,