[codex] improve keyword and hybrid retrieval

.gitignore

@@ -45,6 +45,7 @@ coverage
 .nyc_output
 playwright-report
 test-results
+packages/cli/benchmarks/local-embeddings/results
 
 # Misc
 *.tgz

apps/web/src/app/(site)/solutions/solutions-page.tsx

@@ -65,6 +65,7 @@ export default function SolutionsPage() {
           query: activeQuery,
           limit: PAGE_SIZE,
           mode: searchMode,
+          ...(searchMode === "keyword" ? { keywordStrategy: "tiered" as const } : {}),
         }),
       ),
     enabled: isSearching,

apps/web/src/components/webmcp-provider.test.ts

@@ -52,10 +52,11 @@ describe("WebMCP tool definitions", () => {
         query: "Next.js cache issue",
         limit: 10,
         mode: "keyword",
+        keywordStrategy: "exact",
       });
       expect(result).toEqual({
         results: [{ id: "1", problem: "test", solution: "fix", score: 0 }],
-        attempts: [{ mode: "keyword", resultCount: 1 }],
+        attempts: [{ mode: "keyword", keywordStrategy: "exact", resultCount: 1 }],
       });
     });
 
@@ -72,6 +73,7 @@ describe("WebMCP tool definitions", () => {
         query: "conceptual miss",
         limit: 10,
         mode: "keyword",
+        keywordStrategy: "exact",
       });
       expect(mocked).toHaveBeenNthCalledWith(2, {
         query: "conceptual miss",
@@ -81,7 +83,7 @@ describe("WebMCP tool definitions", () => {
       expect(result).toEqual({
         results: [{ id: "2", problem: "hybrid", solution: "fix", score: 1 }],
         attempts: [
-          { mode: "keyword", resultCount: 0 },
+          { mode: "keyword", keywordStrategy: "exact", resultCount: 0 },
           { mode: "hybrid", resultCount: 1 },
         ],
       });

apps/web/src/components/webmcp-provider.test.tsx

@@ -55,10 +55,11 @@ describe("WebMCP tool definitions", () => {
         query: "Next.js cache issue",
         limit: 10,
         mode: "keyword",
+        keywordStrategy: "exact",
       });
       expect(result).toEqual({
         results: [{ id: "1", problem: "test", solution: "fix", score: 0 }],
-        attempts: [{ mode: "keyword", resultCount: 1 }],
+        attempts: [{ mode: "keyword", keywordStrategy: "exact", resultCount: 1 }],
       });
     });
 
@@ -75,6 +76,7 @@ describe("WebMCP tool definitions", () => {
         query: "conceptual miss",
         limit: 10,
         mode: "keyword",
+        keywordStrategy: "exact",
       });
       expect(mockFn).toHaveBeenNthCalledWith(2, {
         query: "conceptual miss",
@@ -84,24 +86,28 @@ describe("WebMCP tool definitions", () => {
       expect(result).toEqual({
         results: [{ id: "2", problem: "hybrid", solution: "fix", score: 1 }],
         attempts: [
-          { mode: "keyword", resultCount: 0 },
+          { mode: "keyword", keywordStrategy: "exact", resultCount: 0 },
           { mode: "hybrid", resultCount: 1 },
         ],
       });
     });
 
     it("auto mode reports fallback failure without dropping keyword miss context", async () => {
       const mockFn = trpcClient.solutions.search.query as ReturnType<typeof vi.fn>;
-      mockFn.mockResolvedValueOnce([]).mockRejectedValueOnce(new Error("Authentication required"));
+      mockFn
+        .mockResolvedValueOnce([])
+        .mockRejectedValueOnce(new Error("Authentication required"))
+        .mockResolvedValueOnce([]);
 
       const tool = WEBMCP_TOOLS.find((candidate) => candidate.name === "search_solutions");
       const result = await tool?.execute({ query: "conceptual miss" });
 
       expect(result).toEqual({
         results: [],
         attempts: [
-          { mode: "keyword", resultCount: 0 },
+          { mode: "keyword", keywordStrategy: "exact", resultCount: 0 },
           { mode: "hybrid", error: "Authentication required" },
+          { mode: "keyword", keywordStrategy: "tiered", resultCount: 0 },
         ],
         message:
           "Keyword search returned no results and hybrid fallback was unavailable. Try one smaller or sharper keyword query before debugging from scratch.",

apps/web/src/components/webmcp-provider.tsx

@@ -35,7 +35,7 @@ declare global {
 const searchSolutionsTool: WebMCPTool = {
   name: "search_solutions",
   description:
-    "Search ClankerOverflow before fresh debugging whenever an error, stack trace, failing command, failing test, CI/build failure, regression, dependency issue, runtime failure, unfamiliar tool behavior, or reusable implementation problem appears. Default auto mode starts with keyword search and tries hybrid after an empty keyword result when available. Use the smallest distinctive keyword fingerprint and tags as relevance signals.",
+    "Search ClankerOverflow before fresh debugging whenever an error, stack trace, failing command, failing test, CI/build failure, regression, dependency issue, runtime failure, unfamiliar tool behavior, or reusable implementation problem appears. Default auto mode tries exact keyword search, then hybrid after a miss, then tiered keyword retrieval if hybrid is unavailable. Use the smallest distinctive keyword fingerprint and tags as relevance signals.",
   inputSchema: {
     type: "object",
     properties: {
@@ -49,7 +49,7 @@ const searchSolutionsTool: WebMCPTool = {
         enum: ["auto", "keyword", "semantic", "hybrid"],
         default: "auto",
         description:
-          "auto: keyword first, then hybrid on empty results when available; keyword, semantic, or hybrid for strict mode",
+          "auto: exact keyword, then hybrid on a miss, then tiered keyword if hybrid is unavailable; keyword uses exact-first relaxed-fill retrieval",
       },
     },
     required: ["query"],
@@ -63,22 +63,26 @@ const searchSolutionsTool: WebMCPTool = {
       : "auto";
 
     try {
-      const search = (searchMode: ConcreteSearchMode) =>
+      const search = (searchMode: ConcreteSearchMode, keywordStrategy?: "exact" | "tiered") =>
         trpcClient.solutions.search.query({
           query,
           limit: 10,
           mode: searchMode,
+          ...(keywordStrategy ? { keywordStrategy } : {}),
         });
 
       if (mode !== "auto") {
-        const results = await search(mode);
+        const results = await search(mode, mode === "keyword" ? "tiered" : undefined);
         return { results, attempts: [{ mode, resultCount: results.length }] };
       }
 
-      const keywordResults = await search("keyword");
-      const attempts: Array<{ mode: ConcreteSearchMode; resultCount?: number; error?: string }> = [
-        { mode: "keyword", resultCount: keywordResults.length },
-      ];
+      const keywordResults = await search("keyword", "exact");
+      const attempts: Array<{
+        mode: ConcreteSearchMode;
+        keywordStrategy?: "exact" | "tiered";
+        resultCount?: number;
+        error?: string;
+      }> = [{ mode: "keyword", keywordStrategy: "exact", resultCount: keywordResults.length }];
       if (keywordResults.length > 0) {
         return { results: keywordResults, attempts };
       }
@@ -92,8 +96,14 @@ const searchSolutionsTool: WebMCPTool = {
           mode: "hybrid",
           error: error instanceof Error ? error.message : "Unknown error",
         });
+        const relaxedResults = await search("keyword", "tiered");
+        attempts.push({
+          mode: "keyword",
+          keywordStrategy: "tiered",
+          resultCount: relaxedResults.length,
+        });
         return {
-          results: keywordResults,
+          results: relaxedResults,
           attempts,
           message:
             "Keyword search returned no results and hybrid fallback was unavailable. Try one smaller or sharper keyword query before debugging from scratch.",

package.json

@@ -14,9 +14,12 @@
     "dev:server": "turbo run dev --filter=server",
     "test": "turbo run test",
     "test:e2e:local": "tsx scripts/test-cli-local-e2e.ts",
+    "benchmark:local-embeddings": "tsx packages/cli/benchmarks/local-embeddings/run.ts",
+    "benchmark:hosted-retrieval": "pnpm --filter @clankeroverflow/db benchmark:hosted-retrieval",
     "db:push": "turbo run db:push --filter=@clankeroverflow/db",
     "db:generate": "turbo run db:generate --filter=@clankeroverflow/db",
     "db:migrate": "turbo run db:migrate --filter=@clankeroverflow/db",
+    "db:migrate:search-indexes": "pnpm --filter @clankeroverflow/db db:migrate:search-indexes",
     "deploy": "turbo run deploy --filter=@clankeroverflow/infra",
     "destroy": "turbo run destroy --filter=@clankeroverflow/infra",
     "release:cli:patch": "pnpm --filter @clankeroverflow/cli version patch --no-git-tag-version --no-git-checks && pnpm --filter @clankeroverflow/cli run build && pnpm run lint:fix && pnpm run format",

packages/api/src/routers/solutions.test.ts

@@ -19,7 +19,7 @@
   chain.where = vi.fn(() => chain);
   chain.orderBy = vi.fn(() => chain);
   chain.limit = vi.fn(() => result);
   chain.then = (resolve: (v: unknown) => unknown) => resolve(result);
   chain[Symbol.iterator] = () => result[Symbol.iterator]();
   return chain;
 }
@@ -131,6 +131,7 @@
       event: "solution searched",
       properties: {
         search_mode: "keyword",
+        keyword_strategy: "exact",
         query_length: 4,
         result_count: 1,
       },

packages/api/src/routers/solutions.ts

@@ -487,6 +487,7 @@ export const solutionsRouter = router({
         query: z.string().min(1, "Search query is required").max(500, "Search query too long"),
         limit: z.number().min(1).max(20).default(1),
         mode: z.enum(["keyword", "semantic", "hybrid"]).default("keyword"),
+        keywordStrategy: z.enum(["exact", "tiered"]).default("exact"),
       }),
     )
     .query(async ({ ctx, input }) => {
@@ -515,7 +516,7 @@ export const solutionsRouter = router({
 
       if (input.mode === "keyword") {
         results = await withTimeout(
-          searchSolutions(ctx.db, payload),
+          searchSolutions(ctx.db, { ...payload, strategy: input.keywordStrategy }),
           DB_TIMEOUT_MS,
           "Solution search timed out",
         );
@@ -567,6 +568,7 @@ export const solutionsRouter = router({
         event: "solution searched",
         properties: {
           search_mode: input.mode,
+          ...(input.mode === "keyword" ? { keyword_strategy: input.keywordStrategy } : {}),
           query_length: trimmed.length,
           result_count: results.length,
         },

packages/api/src/semantic/search.test.ts

@@ -127,4 +127,32 @@ describe("searchSolutionsHybrid", () => {
     expect(out.map((r) => r.id)).toEqual(["b", "a", "c"]);
     expect(db.execute).toHaveBeenCalledTimes(1);
   });
+
+  test("can promote agreement between semantic and keyword ranks with RRF", async () => {
+    const db = createDb({
+      semanticRows: [createRow("a"), createRow("b")],
+      keywordRows: [createRow("b"), createRow("c")],
+    });
+    const ai = { run: vi.fn(async () => ({ data: [[0.1]] })) };
+    const vectorize = {
+      query: vi.fn(async () => ({
+        matches: [
+          { id: "a", score: 0.9 },
+          { id: "b", score: 0.8 },
+        ],
+      })),
+      upsert: vi.fn(async () => {}),
+    };
+
+    const out = await searchSolutionsHybrid({
+      db: db as any,
+      ai,
+      vectorize,
+      query: "cache invalidation",
+      limit: 3,
+      fusion: "rrf",
+    });
+
+    expect(out.map((row) => row.id)).toEqual(["b", "c", "a"]);
+  });
 });

packages/api/src/semantic/search.ts

@@ -9,6 +9,9 @@ import type { WorkersAiBinding } from "./embeddings";
 import { embedTexts } from "./embeddings";
 import { solutionEmbeddingText } from "./solution-text";
 
+export type HostedHybridFusion = "semantic-first" | "rrf";
+export const HOSTED_HYBRID_FUSION: HostedHybridFusion = "semantic-first";
+
 /** Subset of Vectorize binding used for solution search. */
 export type SolutionVectorizeBinding = {
   query(
@@ -73,6 +76,7 @@ export async function searchSolutionsHybrid(params: {
   vectorize: SolutionVectorizeBinding;
   query: string;
   limit: number;
+  fusion?: HostedHybridFusion;
 }): Promise<SolutionRow[]> {
   const q = params.query.trim();
   if (!q) return [];
@@ -83,11 +87,39 @@ export async function searchSolutionsHybrid(params: {
       ai: params.ai,
       vectorize: params.vectorize,
       query: q,
-      limit: params.limit,
+      limit: Math.max(params.limit, 20),
+    }),
+    searchSolutions(params.db, {
+      query: q,
+      limit: Math.max(params.limit, 20),
+      strategy: "tiered",
     }),
-    searchSolutions(params.db, { query: q, limit: params.limit }),
   ]);
 
+  if ((params.fusion ?? HOSTED_HYBRID_FUSION) === "rrf") {
+    const k = 60;
+    const scores = new Map<string, { row: SolutionRow; score: number; bestRank: number }>();
+    for (const [rows, weight] of [
+      [keywordRows, 1.25],
+      [semanticOrdered, 1],
+    ] as const) {
+      rows.forEach((row, index) => {
+        const rank = index + 1;
+        const existing = scores.get(row.id);
+        if (existing) {
+          existing.score += weight / (k + rank);
+          existing.bestRank = Math.min(existing.bestRank, rank);
+        } else {
+          scores.set(row.id, { row, score: weight / (k + rank), bestRank: rank });
+        }
+      });
+    }
+    return [...scores.values()]
+      .sort((a, b) => b.score - a.score || b.row.score - a.row.score || a.bestRank - b.bestRank)
+      .slice(0, params.limit)
+      .map((entry) => entry.row);
+  }
+
   const byId = new Map<string, SolutionRow>();
   for (const r of keywordRows) byId.set(r.id, r);
   for (const r of semanticOrdered) byId.set(r.id, r);

packages/cli/benchmarks/local-embeddings/README.md

@@ -0,0 +1,58 @@
+# Local Embedding Benchmark
+
+This benchmark compares local GGUF embedding configurations through the same document formatting,
+token chunking, SQLite FTS5, sqlite-vec cosine search, and reciprocal-rank fusion used by local mode.
+It never reads or writes the user's ClankerOverflow database and does not change the shipped default.
+
+## Run
+
+Run the complete quality and performance protocol:
+
+```sh
+pnpm benchmark:local-embeddings
+```
+
+Useful focused runs:
+
+```sh
+pnpm benchmark:local-embeddings --models bge --backend cpu --quality-only
+pnpm benchmark:local-embeddings --models qwen,nomic --performance-only --repetitions 3
+pnpm benchmark:local-embeddings --report-from results/run.json --output results/run.json
+```
+
+The runner supports `--models`, `--backend cpu|auto`, `--quality-only`, `--performance-only`,
+`--repetitions`, `--cold-repetitions`, and `--output`. Models are stored below the repository's
+ignored `.cache` directory by default. Set `CLANKER_BENCHMARK_MODEL_CACHE` to override it.
+
+Every model URL contains an immutable repository revision, and every download is checked against
+the SHA-256 and byte size recorded in `models.ts`. Results are written as raw JSON and a Markdown
+report. The default `results` directory is ignored because measurements are host-specific.
+
+The quality report includes separate exact and tiered keyword baselines. Exact requires all query
+terms; tiered keeps exact matches first and fills remaining slots from relaxed prefix-OR matches.
+Hybrid uses the same relaxed lexical candidate pool as production.
+
+## Hosted promotion benchmark
+
+Run the disposable Workers AI + Vectorize + PostgreSQL benchmark with Cloudflare credentials and a
+Postgres URL whose role can create temporary databases:
+
+```sh
+DATABASE_URL='postgresql://...' pnpm benchmark:hosted-retrieval
+```
+
+The command creates a uniquely staged Alchemy app, a 768-dimensional cosine Vectorize index, and a
+temporary Postgres database. Cleanup runs in `finally`; a precise manual destroy command is printed
+if Cloudflare cleanup fails. RRF passes only when overall nDCG@10 improves by at least 0.01, MRR@10
+and Recall@10 decline by no more than 0.005, and no category or language slice loses over 0.03
+nDCG@10. Hosted RRF remains disabled until a run passes this gate.
+
+## Protocol
+
+- Corpus: 200 sanitized English solution entries and 100 single-reviewer queries.
+- Queries: 80 English plus 10 French-to-English and 10 Arabic-to-English searches.
+- Quality: nDCG@10, MRR@10, and Recall@1/3/10 with seeded bootstrap confidence intervals.
+- Performance: process-cold load and first query, steady indexing throughput, warm semantic query
+  p50/p95, artifact size, and peak RSS.
+- Profiles: model-native prompts plus raw-text quality ablations for Qwen and Nomic.
+- Lanes: CPU-only and the unchanged node-llama-cpp auto-selected backend.