diff --git a/src/lib/db/migrate.ts b/src/lib/db/migrate.ts index bb69ead..243b7f7 100644 --- a/src/lib/db/migrate.ts +++ b/src/lib/db/migrate.ts @@ -331,4 +331,53 @@ export async function runMigrations(): Promise { console.error("[migrate] Migration failed:", err); throw err; } + + // ─── Semantic Cache (pgvector) ────────────────────────────────────────────── + // Graceful: if pgvector ext not available, log warning and skip — system still works + try { + await sql`CREATE EXTENSION IF NOT EXISTS vector`; + await sql` + CREATE TABLE IF NOT EXISTS semantic_cache ( + id BIGSERIAL PRIMARY KEY, + query_hash TEXT UNIQUE NOT NULL, + query TEXT NOT NULL, + embedding vector(768), + response JSONB NOT NULL, + provider TEXT, + model TEXT, + hit_count INT DEFAULT 0, + created_at TIMESTAMPTZ DEFAULT now(), + last_used_at TIMESTAMPTZ DEFAULT now() + ) + `; + await sql` + CREATE INDEX IF NOT EXISTS idx_semantic_cache_embedding + ON semantic_cache USING ivfflat (embedding vector_cosine_ops) + WITH (lists = 100) + `; + await sql`CREATE INDEX IF NOT EXISTS idx_semantic_cache_used ON semantic_cache(last_used_at DESC)`; + console.log("[migrate] pgvector + semantic_cache ready"); + } catch (err) { + console.warn("[migrate] pgvector not available — semantic cache disabled:", String(err).slice(0, 150)); + } + + // ─── Performance indexes ───────────────────────────────────────────────────── + // Note: CREATE INDEX CONCURRENTLY cannot run inside transaction, + // so we use normal CREATE INDEX IF NOT EXISTS here + await sql` + CREATE INDEX IF NOT EXISTS idx_health_cooldown_active + ON health_logs(model_id, cooldown_until) + WHERE cooldown_until > now() + `.catch((err) => console.warn("[migrate] idx_health_cooldown_active:", String(err).slice(0, 100))); + + await sql` + CREATE INDEX IF NOT EXISTS idx_gateway_recent + ON gateway_logs(created_at DESC, status) + `.catch((err) => console.warn("[migrate] idx_gateway_recent:", String(err).slice(0, 100))); + + await sql` + CREATE INDEX IF NOT EXISTS idx_exam_passed_recent + ON exam_attempts(model_id, started_at DESC) + WHERE passed = true + `.catch((err) => console.warn("[migrate] idx_exam_passed_recent:", String(err).slice(0, 100))); } diff --git a/src/lib/semantic-cache.ts b/src/lib/semantic-cache.ts new file mode 100644 index 0000000..382da0b --- /dev/null +++ b/src/lib/semantic-cache.ts @@ -0,0 +1,140 @@ +/** + * Semantic Cache — store responses indexed by message embedding + * + * Lookup: cosine similarity > threshold → return cached response + * Store: embed query + response, persist to pgvector table + * + * Embedding source: Ollama local `nomic-embed-text` (free, no external API) + * Fallback: if Ollama or pgvector unavailable, operations no-op gracefully + */ +import { getSqlClient } from "@/lib/db/schema"; +import { createHash } from "crypto"; + +const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434"; +const EMBED_MODEL = "nomic-embed-text"; +const DEFAULT_THRESHOLD = 0.92; +const EMBED_TIMEOUT_MS = 3_000; + +export interface SemanticHit { + query: string; + response: Record; + provider: string | null; + model: string | null; + similarity: number; + hit_count: number; +} + +function hashQuery(query: string): string { + return createHash("sha256").update(query).digest("hex").slice(0, 32); +} + +async function embedQuery(query: string): Promise { + try { + const res = await fetch(`${OLLAMA_BASE_URL}/api/embeddings`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: EMBED_MODEL, prompt: query }), + signal: AbortSignal.timeout(EMBED_TIMEOUT_MS), + }); + if (!res.ok) return null; + const json = await res.json(); + const embedding = json.embedding as number[] | undefined; + return Array.isArray(embedding) && embedding.length > 0 ? embedding : null; + } catch { + return null; + } +} + +function vectorLiteral(vec: number[]): string { + return "[" + vec.join(",") + "]"; +} + +/** + * Look up cached response by semantic similarity. + * Returns null if no match or any error (graceful fallback). + */ +export async function getCachedBySimilarity( + query: string, + threshold: number = DEFAULT_THRESHOLD +): Promise { + if (!query || query.length < 10) return null; + const embedding = await embedQuery(query); + if (!embedding) return null; + try { + const sql = getSqlClient(); + const vecStr = vectorLiteral(embedding); + const rows = await sql; + provider: string | null; + model: string | null; + similarity: number; + hit_count: number; + }>>` + SELECT query, response, provider, model, hit_count, + 1 - (embedding <=> ${vecStr}::vector) AS similarity + FROM semantic_cache + ORDER BY embedding <=> ${vecStr}::vector + LIMIT 1 + `; + const row = rows[0]; + if (!row || row.similarity < threshold) return null; + // Update hit count + last_used_at (fire and forget) + sql` + UPDATE semantic_cache + SET hit_count = hit_count + 1, last_used_at = now() + WHERE query = ${row.query} + `.catch(() => {}); + return row; + } catch { + return null; + } +} + +/** + * Store response with embedding. Idempotent via query_hash. + */ +export async function storeSemanticResponse( + query: string, + response: Record, + provider: string | null, + model: string | null +): Promise { + if (!query || query.length < 10) return; + const embedding = await embedQuery(query); + if (!embedding) return; + try { + const sql = getSqlClient(); + const hash = hashQuery(query); + const vecStr = vectorLiteral(embedding); + await sql` + INSERT INTO semantic_cache (query_hash, query, embedding, response, provider, model) + VALUES (${hash}, ${query.slice(0, 2000)}, ${vecStr}::vector, ${JSON.stringify(response)}::jsonb, ${provider}, ${model}) + ON CONFLICT (query_hash) DO UPDATE SET + last_used_at = now(), + hit_count = semantic_cache.hit_count + 1 + `; + } catch { + // silent — cache is optional + } +} + +export async function getSemanticCacheStats(): Promise<{ + total: number; + hits: number; + avgHits: number; +}> { + try { + const sql = getSqlClient(); + const rows = await sql<{ total: number; hits: number; avg: number }[]>` + SELECT COUNT(*)::int as total, + COALESCE(SUM(hit_count), 0)::int as hits, + COALESCE(AVG(hit_count), 0)::float as avg + FROM semantic_cache + `; + const r = rows[0]; + return { total: r?.total ?? 0, hits: r?.hits ?? 0, avgHits: r?.avg ?? 0 }; + } catch { + return { total: 0, hits: 0, avgHits: 0 }; + } +}