fix: wire advanced FTS5 parser into local keyword search

packages/cli/commands/search-solutions.md

@@ -11,7 +11,7 @@ Search ClankerOverflow for solutions matching the query. Use this as the first s
 
 Keep keyword queries short. Start with the smallest distinctive literal fingerprint: an error code, command, package, or short sanitized error phrase. Use tags as relevance signals. Add one package, command, or tag only if the first search is too broad.
 
-**Advanced keyword syntax (local FTS5)**: in keyword/hybrid/auto mode, a query may use FTS5 operators when it contains them, e.g. `database AND crash`, `"oauth callback" OR react*`, `tags:react hooks`, `database NOT physics`, `(a OR b) AND c`, or `NEAR(token, nft, 5)`. Unknown columns, unbalanced parentheses, doubled operators, or stray operators are rejected with a clear message. To search for operator words literally (e.g. the literal text `AND`), wrap the whole query in double quotes.
+**Advanced keyword syntax (local FTS5)**: in keyword/hybrid/auto mode, a query may use FTS5 operators when it contains them, e.g. `database AND crash`, `"oauth callback" OR react*`, `tags:react hooks`, `database NOT physics`, `(a OR b) AND c`, or `NEAR(token nft, 5)`. Unknown columns, unbalanced parentheses, doubled operators, or stray operators are rejected with a clear message. To search for operator words literally (e.g. the literal text `AND`), wrap the whole query in double quotes.
 
 **Negative/leading-dash values**: to search for a query that itself starts with `-` (a negative number, a version string like `v2.0-beta-1`), separate options from the query with `--`, e.g. `clanker search -- -1`.
 

packages/cli/src/index.ts

@@ -18,7 +18,7 @@ import {
 import { createSolutionBackend } from "./mcp/create-backend.js";
 import { startMcpServer } from "./mcp/server.js";
 import { formatSearchResults } from "./mcp/format.js";
-import { LocalBackend } from "./mcp/local-backend.js";
+import { FtsQuerySyntaxError, LocalBackend } from "./mcp/local-backend.js";
 import { downloadDefaultLocalModel } from "./mcp/local-semantic.js";
 import { hasSetupFailures, setupAgents, type Agent, type SkillSelection } from "./setup.js";
 import pc from "picocolors";
@@ -322,7 +322,11 @@ export function createProgram(options: CreateProgramOptions = {}) {
           source: backendMode,
         });
       } catch (error: any) {
-        console.error(pc.red(pc.bold("✖ Error searching solutions:")));
+        if (error instanceof FtsQuerySyntaxError) {
+          console.error(pc.red(pc.bold("✖ Invalid search syntax:")));
+        } else {
+          console.error(pc.red(pc.bold("✖ Error searching solutions:")));
+        }
         console.error(pc.red(error.message || error));
         process.exit(1);
       }
@@ -567,7 +571,11 @@ export function createProgram(options: CreateProgramOptions = {}) {
           source: "local",
         });
       } catch (error: any) {
-        console.error(pc.red(pc.bold("✖ Error searching local solutions:")));
+        if (error instanceof FtsQuerySyntaxError) {
+          console.error(pc.red(pc.bold("✖ Invalid search syntax:")));
+        } else {
+          console.error(pc.red(pc.bold("✖ Error searching local solutions:")));
+        }
         console.error(pc.red(error.message || error));
         process.exit(1);
       }

packages/cli/src/mcp/local-backend.test.ts

@@ -374,6 +374,70 @@ describe("CLI local MCP backend", () => {
       "search query must not be empty",
     );
   });
+
+  test("exact keyword search honors the AND operator", async () => {
+    const backend = new LocalBackend(dbPath);
+    const { id } = await backend.log({
+      problem: "OAuth callback timeout",
+      solution: "Keep waitUntil tasks alive",
+      tags: "auth",
+    });
+    await backend.log({
+      problem: "OAuth misconfiguration only",
+      solution: "Check redirect URIs.",
+      tags: "auth",
+    });
+
+    const results = await backend.search({
+      query: "OAuth AND timeout",
+      limit: 5,
+      mode: "keyword",
+      keywordStrategy: "exact",
+    });
+    expect(results).toHaveLength(1);
+    expect(results[0]!.id).toBe(id);
+  });
+
+  test("exact keyword search honors column filters", async () => {
+    const backend = new LocalBackend(dbPath);
+    await backend.log({
+      problem: "OAuth callback timeout",
+      solution: "Keep waitUntil tasks alive",
+      tags: "auth",
+    });
+    await backend.log({
+      problem: "Startup race condition",
+      solution: "Await initialization.",
+      tags: "init",
+    });
+
+    const results = await backend.search({
+      query: "tags:auth",
+      limit: 5,
+      mode: "keyword",
+      keywordStrategy: "exact",
+    });
+    expect(results).toHaveLength(1);
+    expect(results[0]!.problem).toBe("OAuth callback timeout");
+  });
+
+  test("exact keyword search rejects a malformed advanced query", async () => {
+    const backend = new LocalBackend(dbPath);
+    await backend.log({
+      problem: "Database crash",
+      solution: "Restart the service.",
+      tags: "db",
+    });
+
+    await expect(
+      backend.search({
+        query: "database AND",
+        limit: 5,
+        mode: "keyword",
+        keywordStrategy: "exact",
+      }),
+    ).rejects.toThrow(FtsQuerySyntaxError);
+  });
 });
 
 describe("ftsQuery", () => {
@@ -396,8 +460,9 @@ describe("ftsQuery", () => {
     expect(ftsQuery("   ")).toBe("");
   });
 
-  test("simple mode rejects bare leading-dash negation", () => {
-    expect(() => ftsQuery("-foo")).toThrow(FtsQuerySyntaxError);
+  test("simple mode treats a leading dash as punctuation, not negation", () => {
+    expect(ftsQuery("-foo")).toBe('"foo"');
+    expect(ftsQuery("sqlite -wal")).toBe('"sqlite" "wal"');
   });
 
   test("advanced mode preserves the AND operator", () => {
@@ -416,6 +481,22 @@ describe("ftsQuery", () => {
     expect(ftsQuery("tags:react hooks")).toBe('tags : "react" AND "hooks"');
   });
 
+  test("advanced mode preserves balanced parentheses as a group", () => {
+    expect(ftsQuery("(database OR crash) AND startup")).toBe(
+      '( "database" OR "crash" ) AND "startup"',
+    );
+  });
+
+  test("advanced mode normalizes NEAR(...) comma form to space-separated FTS5", () => {
+    expect(ftsQuery("NEAR(token, nft, 5)")).toBe("NEAR(token nft, 5)");
+    expect(ftsQuery("NEAR(oauth timeout)")).toBe("NEAR(oauth timeout)");
+    expect(ftsQuery("NEAR(token, nft)")).toBe("NEAR(token nft)");
+  });
+
+  test("advanced mode rejects an unterminated NEAR(...) expression", () => {
+    expect(() => ftsQuery("NEAR(token nft")).toThrow(FtsQuerySyntaxError);
+  });
+
   test("advanced mode rejects unknown column filters", () => {
     expect(() => ftsQuery("foo:bar")).toThrow(FtsQuerySyntaxError);
   });

packages/cli/src/mcp/local-backend.ts

@@ -37,29 +37,6 @@ function nowIso() {
   return new Date().toISOString();
 }
 
-export function localFtsQuery(query: string) {
-  const normalized = query
-    .replace(/https?:\/\/\S+/gi, (match) => match.replace(/[/:.?=&%#-]+/g, " "))
-    .trim();
-  const phrases = [...normalized.matchAll(/"([^"]+)"/g)].map((match) => match[1]?.trim() ?? "");
-  const withoutPhrases = normalized.replace(/"[^"]+"/g, " ");
-  const terms = withoutPhrases.match(/[\p{L}\p{N}_][\p{L}\p{N}_./:-]*/gu) ?? [];
-  const phraseClauses = phrases
-    .map((phrase) => phrase.replace(/[^\p{L}\p{N}_-]+/gu, " ").trim())
-    .filter(Boolean)
-    .map((phrase) => `"${phrase.replaceAll('"', '""')}"`);
-  const termClauses = terms
-    .flatMap((term) =>
-      term
-        .replace(/[^\p{L}\p{N}_-]+/gu, " ")
-        .trim()
-        .split(/\s+/),
-    )
-    .filter(Boolean)
-    .map((term) => `"${term.replaceAll('"', '""')}"`);
-  return [...new Set([...phraseClauses, ...termClauses])].join(" ");
-}
-
 export class FtsQuerySyntaxError extends Error {}
 
 /** FTS5 column names backed by the solution_fts(problem, solution, tags) table. */
@@ -70,7 +47,7 @@ const FTS_BOOLEAN_OPERATORS = new Set(["AND", "OR", "NOT"]);
 function ftsSyntaxError(message: string): never {
   throw new FtsQuerySyntaxError(
     `${message} Valid FTS5 syntax: quoted "phrases", prefix term*, ` +
-      `column:term, term AND/OR/NOT term, (group), NEAR(term, term, N). ` +
+      `column:term, term AND/OR/NOT term, (group), NEAR(term term [, N]). ` +
       `To search for these words literally, wrap the whole query in double quotes.`,
   );
 }
@@ -111,7 +88,8 @@ function tokenizeFtsQuery(query: string): FtsToken[] {
     }
   };
 
-  for (const char of query) {
+  for (let i = 0; i < query.length; i += 1) {
+    const char = query[i]!;
     if (inQuotes) {
       current += char;
       if (char === '"') inQuotes = false;
@@ -122,6 +100,17 @@ function tokenizeFtsQuery(query: string): FtsToken[] {
       inQuotes = true;
       continue;
     }
+    // Capture a whole NEAR(...) expression as one token so the comma-separated
+    // form (e.g. NEAR(token, nft, 5)) survives as a unit and can be normalized
+    // to FTS5's space-separated form. Parens inside quotes are already handled
+    // above by the inQuotes branch.
+    if (char === "(" && /^NEAR$/i.test(current.trim())) {
+      const near = captureNear(query, i);
+      current = "";
+      tokens.push({ type: "word", value: near.value });
+      i = near.endIndex;
+      continue;
+    }
     if (char === "(" || char === ")") {
       flushWord();
       tokens.push(char === "(" ? { type: "lparen" } : { type: "rparen" });
@@ -138,6 +127,51 @@ function tokenizeFtsQuery(query: string): FtsToken[] {
   return tokens;
 }
 
+/**
+ * Normalize a `NEAR(...)` expression to FTS5's space-separated form. FTS5 wants
+ * `NEAR(term1 term2 [, N])` where terms are space-separated and the optional
+ * integer distance follows a comma. The forgiving comma-separated form
+ * `NEAR(a, b, 5)` is translated by replacing term-separating commas with spaces
+ * while preserving a trailing `, N` distance argument.
+ */
+function normalizeNear(raw: string): string {
+  const match = raw.match(/^NEAR\s*\((.*)\)$/is);
+  if (!match) ftsSyntaxError("malformed NEAR(...) expression.");
+  const inner = match[1]!.trim();
+  if (!inner) ftsSyntaxError("NEAR(...) needs at least one term.");
+  // Split on commas, drop empties, so "a, b, 5" -> ["a","b","5"] and "a b, 5" -> ["a b","5"].
+  const parts = inner
+    .split(",")
+    .map((part) => part.trim())
+    .filter(Boolean);
+  // A trailing bare integer is the NEAR distance; keep it after a comma.
+  const last = parts[parts.length - 1];
+  const distance = parts.length >= 2 && /^\d+$/.test(last!) ? `, ${last}` : "";
+  const terms = (distance ? parts.slice(0, -1) : parts).join(" ").trim();
+  if (!terms) ftsSyntaxError("NEAR(...) needs at least one term.");
+  return `NEAR(${terms}${distance})`;
+}
+
+/**
+ * Capture a `NEAR(...)` expression starting at the `(` located at `parenIndex`,
+ * returning the raw text (including `NEAR(` ... `)`) and the index of the final
+ * `)`. Terms inside are left as-is; the caller normalizes commas to spaces.
+ */
+function captureNear(query: string, parenIndex: number): { value: string; endIndex: number } {
+  let depth = 0;
+  let j = parenIndex;
+  for (; j < query.length; j += 1) {
+    const char = query[j]!;
+    if (char === "(") depth += 1;
+    else if (char === ")") {
+      depth -= 1;
+      if (depth === 0) break;
+    }
+  }
+  if (depth !== 0) ftsSyntaxError('unmatched "(" in NEAR(...) expression.');
+  return { value: query.slice(parenIndex - 4, j + 1), endIndex: j };
+}
+
 /** Convert a quoted `"..."` token (with surrounding quotes) into a phrase token. */
 function phraseToken(raw: string): FtsToken {
   // Strip the outer quotes; inner content is the literal phrase.
@@ -250,7 +284,7 @@ function buildAdvancedQuery(tokens: FtsToken[]) {
         if (!expectTerm) parts.push("AND"); // implicit AND between adjacent terms
         parts.push(
           token.type === "word" && /^NEAR\s*\(/i.test(token.value)
-            ? token.value
+            ? normalizeNear(token.value)
             : renderAdvancedTerm(token),
         );
         expectTerm = false;
@@ -282,15 +316,14 @@ function buildSimpleQuery(query: string) {
         `Drop them or use explicit AND/OR/NOT operators.`,
     );
   }
-  const terms = withoutPhrases.match(/-?[\p{L}\p{N}_][\p{L}\p{N}_./:-]*/gu) ?? [];
+  const terms = withoutPhrases.match(/[\p{L}\p{N}_][\p{L}\p{N}_./:-]*/gu) ?? [];
   const rendered = [...phrases, ...terms]
     .map((term) => {
-      if (term.startsWith("-")) {
-        ftsSyntaxError(
-          `bare negation "-${term.slice(1)}" needs a positive term first; use FTS5 NOT instead.`,
-        );
-      }
-      const clean = term.replace(/[^\p{L}\p{N}_-]+/gu, " ").trim();
+      // Treat a leading hyphen (e.g. "sqlite -wal") as punctuation, not FTS5
+      // NOT: strip it so the term still matches. Bare negation must be spelled
+      // out explicitly with the FTS5 NOT operator in advanced mode.
+      const stripped = term.replace(/^-+/, "");
+      const clean = stripped.replace(/[^\p{L}\p{N}_-]+/gu, " ").trim();
       return clean ? quoteFtsTerm(clean) : "";
     })
     .filter(Boolean);
@@ -387,7 +420,7 @@ function searchLocalKeywordExpression(db: LocalDb, query: string, limit: number)
 }
 
 export function searchLocalKeywordExact(db: LocalDb, queryText: string, limit: number) {
-  return searchLocalKeywordExpression(db, localFtsQuery(queryText.trim()), limit);
+  return searchLocalKeywordExpression(db, ftsQuery(queryText.trim()), limit);
 }
 
 export function searchLocalKeywordRelaxed(db: LocalDb, queryText: string, limit: number) {

packages/cli/src/mcp/server.ts

@@ -9,7 +9,11 @@ import type { SolutionBackend } from "./backend.js";
 import { modeForSource, resolveConfig, type ServerConfig } from "./config.js";
 import { createSolutionBackend } from "./create-backend.js";
 import { formatSearchResults } from "./format.js";
-import { LocalBackend, LocalSemanticSearchNotConfiguredError } from "./local-backend.js";
+import {
+  FtsQuerySyntaxError,
+  LocalBackend,
+  LocalSemanticSearchNotConfiguredError,
+} from "./local-backend.js";
 
 const logger = new McpLogger({ name: packageJson.name });
 
@@ -157,6 +161,14 @@ export function createMcpServer(config: ServerConfig = resolveConfig()) {
           });
           return { content: [{ type: "text" as const, text: error.message }] };
         }
+        if (error instanceof FtsQuerySyntaxError) {
+          logger.warn("Invalid FTS5 search syntax", {
+            error: error.message,
+            query,
+            mode,
+          });
+          return { content: [{ type: "text" as const, text: error.message }] };
+        }
         logger.error("search_solutions failed", {
           error: error instanceof Error ? error.message : String(error),
           query,