Optimize Project Finder retrieval performance

db/lab_inventory.py

@@ -30,7 +30,7 @@
 )
 CODE_SAMPLE_TITLE_RE = re.compile(r"\bcode\s+sample\b", re.IGNORECASE)
 MIN_PROJECT_TEXT_CHARS = 90
-PROJECT_FINDER_CHUNK_SOURCE_LIMIT = 1200
+PROJECT_FINDER_CHUNK_SOURCE_LIMIT = 5000
 PROJECT_FINDER_INTELLIGENCE_SOURCE_LIMIT = 400
 
 GENERIC_PART_PATTERNS: tuple[tuple[re.Pattern[str], str, str], ...] = (
@@ -270,14 +270,15 @@ def find(self, user_id: int, *, limit: int = 24, offset: int = 0, candidate_filt
         with self.database.connection() as conn:
             chunk_rows = conn.execute(
                 load_query("project_finder_chunk_candidates.sql"),
-                (terms, PROJECT_FINDER_CHUNK_SOURCE_LIMIT),
+                (self._term_regex(terms), PROJECT_FINDER_CHUNK_SOURCE_LIMIT),
             ).fetchall()
             intelligence_rows = conn.execute(
                 load_query("project_finder_intelligence_candidates.sql"),
                 (terms, PROJECT_FINDER_INTELLIGENCE_SOURCE_LIMIT),
             ).fetchall()
 
         inventory_index = self._inventory_index(inventory, term_rows)
+        chunk_rows = self._annotate_chunk_matches(chunk_rows, self._prepared_search_terms(terms))
         candidates = [
             self._chunk_candidate(row, inventory_index)
             for row in chunk_rows
@@ -348,6 +349,50 @@ def _search_terms(self, rows: list[dict[str, Any]]) -> list[str]:
                 terms.setdefault(term, term)
         return list(terms.values())[:300]
 
+    def _term_regex(self, terms: list[str]) -> str:
+        patterns = []
+        for term in terms:
+            words = re.findall(r"[a-z0-9]+", normalize_part_name(term))
+            if not words:
+                continue
+            pattern = r"[^[:alnum:]]+".join(re.escape(word) for word in words)
+            if len(pattern) >= 2:
+                patterns.append(pattern)
+        if not patterns:
+            return r"a^"
+        deduped = sorted(set(patterns), key=lambda value: (-len(value), value))
+        return r"(^|[^[:alnum:]])(" + "|".join(deduped) + r")([^[:alnum:]]|$)"
+
+    def _prepared_search_terms(self, terms: list[str]) -> list[tuple[str, str]]:
+        prepared = OrderedDict()
+        for term in terms:
+            normalized = normalize_part_name(term)
+            if not normalized:
+                continue
+            prepared.setdefault(normalized, (normalized, compact_part_key(normalized)))
+        return list(prepared.values())
+
+    def _annotate_chunk_matches(self, rows: list[dict[str, Any]], terms: list[tuple[str, str]]) -> list[dict[str, Any]]:
+        annotated = []
+        for row in rows:
+            payload = dict(row)
+            matched_terms = self._matched_terms_for_text(payload.get("chunk_text") or "", terms)
+            if not matched_terms:
+                continue
+            payload["matched_terms"] = matched_terms
+            payload["matched_count"] = len(matched_terms)
+            annotated.append(payload)
+        return annotated
+
+    def _matched_terms_for_text(self, text: str, terms: list[tuple[str, str]]) -> list[str]:
+        normalized_text = normalize_part_name(text)
+        compact_text = compact_part_key(text)
+        matched = []
+        for normalized_term, compact_term in terms:
+            if normalized_term in normalized_text or (compact_term and compact_term in compact_text):
+                matched.append(normalized_term)
+        return matched
+
     def _inventory_index(self, inventory: list[dict[str, Any]], term_rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
         index: dict[str, dict[str, Any]] = {}
         for part in inventory:

db/migrations/067_project_finder_retrieval_indexes.sql

@@ -0,0 +1,20 @@
+BEGIN;
+
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+CREATE INDEX IF NOT EXISTS document_chunks_project_finder_text_trgm_idx
+    ON document_chunks USING gin (lower(chunk_text) gin_trgm_ops);
+
+CREATE INDEX IF NOT EXISTS document_chunks_project_finder_section_trgm_idx
+    ON document_chunks USING gin (lower(coalesce(section_title, '')) gin_trgm_ops);
+
+CREATE INDEX IF NOT EXISTS document_intelligence_project_finder_text_trgm_idx
+    ON document_intelligence USING gin (
+        lower(coalesce(component_name, '') || ' ' || coalesce(component_type, '') || ' ' || coalesce(summary, '')) gin_trgm_ops
+    );
+
+INSERT INTO schema_migrations (version, name)
+VALUES (67, 'project_finder_retrieval_indexes')
+ON CONFLICT (version) DO NOTHING;
+
+COMMIT;

db/queries/project_finder_chunk_candidates.sql

@@ -1,51 +1,34 @@
-WITH terms AS (
-    SELECT DISTINCT lower(term) AS term
-    FROM unnest(%s::text[]) AS term
-    WHERE length(trim(term)) >= 2
-),
-matched AS (
+SELECT source_path,
+       display_name,
+       chunk_index,
+       page_number,
+       section_title,
+       category,
+       quality_score,
+       chunk_text,
+       ARRAY[]::text[] AS matched_terms,
+       0 AS matched_count
+FROM (
     SELECT d.source_path,
            d.display_name,
            dc.chunk_index,
            dp.page_number,
            dc.section_title,
            dc.category,
            dc.quality_score,
-           dc.chunk_text,
-           array_agg(DISTINCT terms.term ORDER BY terms.term) AS matched_terms,
-           count(DISTINCT terms.term) AS matched_count
+           dc.chunk_text
     FROM document_chunks dc
     JOIN documents d ON d.id = dc.document_id
     LEFT JOIN document_pages dp ON dp.id = dc.page_id
-    JOIN terms ON lower(dc.chunk_text) LIKE '%%' || terms.term || '%%'
     WHERE d.status_id = 3
+      AND lower(dc.chunk_text) ~ %s
       AND (
           lower(dc.chunk_text) ~ '(project|experiment|circuit|build|breadboard|schematic|parts|components|wire|wiring)'
           OR dc.category IN ('MED_LEVEL_DETAIL', 'TECH_LEVEL_DETAIL', 'CODE_SAMPLE')
-          OR lower(dc.section_title) LIKE 'code sample%%'
+          OR lower(coalesce(dc.section_title, '')) LIKE 'code sample%%'
       )
-    GROUP BY d.source_path,
-             d.display_name,
-             dc.chunk_index,
-             dp.page_number,
-             dc.section_title,
-             dc.category,
-             dc.quality_score,
-             dc.chunk_text
-)
-SELECT source_path,
-       display_name,
-       chunk_index,
-       page_number,
-       section_title,
-       category,
-       quality_score,
-       chunk_text,
-       matched_terms,
-       matched_count
-FROM matched
-ORDER BY matched_count DESC,
-         quality_score DESC,
+) AS matched
+ORDER BY quality_score DESC,
          source_path,
          page_number NULLS LAST,
          chunk_index

tests/test_lab_inventory.py

@@ -162,6 +162,26 @@ def test_chunk_candidate_uses_inventory_aliases_in_objective(self):
         self.assertIn("NE555 timer -> LM555", candidate["objective"])
         self.assertIn("Breadboard", {part["name"] for part in candidate["missingParts"]})
 
+    def test_chunk_match_annotation_uses_prepared_terms(self):
+        store = ProjectFinderStore(None, None)
+        terms = store._prepared_search_terms(["NE555 timer", "10 segment LED", "missing part"])
+        rows = [
+            {
+                "chunk_text": "Build an NE-555 timer with a 10-segment LED display.",
+                "source_path": "book.pdf",
+            },
+            {
+                "chunk_text": "Copyright and preface text.",
+                "source_path": "preface.pdf",
+            },
+        ]
+
+        annotated = store._annotate_chunk_matches(rows, terms)
+
+        self.assertEqual(len(annotated), 1)
+        self.assertEqual(annotated[0]["matched_terms"], ["ne555 timer", "10 segment led"])
+        self.assertEqual(annotated[0]["matched_count"], 2)
+
     def test_missing_part_summary_ranks_repeated_gaps(self):
         store = ProjectFinderStore(None, None)
         summary = store._missing_part_summary(