From a1d4f129f1d505f1af14b1a857c90cb7dafa5289 Mon Sep 17 00:00:00 2001 From: Romonaga Date: Sat, 13 Jun 2026 13:08:45 -0400 Subject: [PATCH] Optimize Project Finder retrieval performance --- db/lab_inventory.py | 49 ++++++++++++++++++- .../067_project_finder_retrieval_indexes.sql | 20 ++++++++ .../project_finder_chunk_candidates.sql | 49 ++++++------------- tests/test_lab_inventory.py | 20 ++++++++ 4 files changed, 103 insertions(+), 35 deletions(-) create mode 100644 db/migrations/067_project_finder_retrieval_indexes.sql diff --git a/db/lab_inventory.py b/db/lab_inventory.py index 6694fc1..5ff56a6 100644 --- a/db/lab_inventory.py +++ b/db/lab_inventory.py @@ -30,7 +30,7 @@ ) CODE_SAMPLE_TITLE_RE = re.compile(r"\bcode\s+sample\b", re.IGNORECASE) MIN_PROJECT_TEXT_CHARS = 90 -PROJECT_FINDER_CHUNK_SOURCE_LIMIT = 1200 +PROJECT_FINDER_CHUNK_SOURCE_LIMIT = 5000 PROJECT_FINDER_INTELLIGENCE_SOURCE_LIMIT = 400 GENERIC_PART_PATTERNS: tuple[tuple[re.Pattern[str], str, str], ...] = ( @@ -270,7 +270,7 @@ def find(self, user_id: int, *, limit: int = 24, offset: int = 0, candidate_filt with self.database.connection() as conn: chunk_rows = conn.execute( load_query("project_finder_chunk_candidates.sql"), - (terms, PROJECT_FINDER_CHUNK_SOURCE_LIMIT), + (self._term_regex(terms), PROJECT_FINDER_CHUNK_SOURCE_LIMIT), ).fetchall() intelligence_rows = conn.execute( load_query("project_finder_intelligence_candidates.sql"), @@ -278,6 +278,7 @@ def find(self, user_id: int, *, limit: int = 24, offset: int = 0, candidate_filt ).fetchall() inventory_index = self._inventory_index(inventory, term_rows) + chunk_rows = self._annotate_chunk_matches(chunk_rows, self._prepared_search_terms(terms)) candidates = [ self._chunk_candidate(row, inventory_index) for row in chunk_rows @@ -348,6 +349,50 @@ def _search_terms(self, rows: list[dict[str, Any]]) -> list[str]: terms.setdefault(term, term) return list(terms.values())[:300] + def _term_regex(self, terms: list[str]) -> str: + patterns = [] + for term in terms: + words = re.findall(r"[a-z0-9]+", normalize_part_name(term)) + if not words: + continue + pattern = r"[^[:alnum:]]+".join(re.escape(word) for word in words) + if len(pattern) >= 2: + patterns.append(pattern) + if not patterns: + return r"a^" + deduped = sorted(set(patterns), key=lambda value: (-len(value), value)) + return r"(^|[^[:alnum:]])(" + "|".join(deduped) + r")([^[:alnum:]]|$)" + + def _prepared_search_terms(self, terms: list[str]) -> list[tuple[str, str]]: + prepared = OrderedDict() + for term in terms: + normalized = normalize_part_name(term) + if not normalized: + continue + prepared.setdefault(normalized, (normalized, compact_part_key(normalized))) + return list(prepared.values()) + + def _annotate_chunk_matches(self, rows: list[dict[str, Any]], terms: list[tuple[str, str]]) -> list[dict[str, Any]]: + annotated = [] + for row in rows: + payload = dict(row) + matched_terms = self._matched_terms_for_text(payload.get("chunk_text") or "", terms) + if not matched_terms: + continue + payload["matched_terms"] = matched_terms + payload["matched_count"] = len(matched_terms) + annotated.append(payload) + return annotated + + def _matched_terms_for_text(self, text: str, terms: list[tuple[str, str]]) -> list[str]: + normalized_text = normalize_part_name(text) + compact_text = compact_part_key(text) + matched = [] + for normalized_term, compact_term in terms: + if normalized_term in normalized_text or (compact_term and compact_term in compact_text): + matched.append(normalized_term) + return matched + def _inventory_index(self, inventory: list[dict[str, Any]], term_rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: index: dict[str, dict[str, Any]] = {} for part in inventory: diff --git a/db/migrations/067_project_finder_retrieval_indexes.sql b/db/migrations/067_project_finder_retrieval_indexes.sql new file mode 100644 index 0000000..6ab75b3 --- /dev/null +++ b/db/migrations/067_project_finder_retrieval_indexes.sql @@ -0,0 +1,20 @@ +BEGIN; + +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE INDEX IF NOT EXISTS document_chunks_project_finder_text_trgm_idx + ON document_chunks USING gin (lower(chunk_text) gin_trgm_ops); + +CREATE INDEX IF NOT EXISTS document_chunks_project_finder_section_trgm_idx + ON document_chunks USING gin (lower(coalesce(section_title, '')) gin_trgm_ops); + +CREATE INDEX IF NOT EXISTS document_intelligence_project_finder_text_trgm_idx + ON document_intelligence USING gin ( + lower(coalesce(component_name, '') || ' ' || coalesce(component_type, '') || ' ' || coalesce(summary, '')) gin_trgm_ops + ); + +INSERT INTO schema_migrations (version, name) +VALUES (67, 'project_finder_retrieval_indexes') +ON CONFLICT (version) DO NOTHING; + +COMMIT; diff --git a/db/queries/project_finder_chunk_candidates.sql b/db/queries/project_finder_chunk_candidates.sql index 055b6a7..d736d56 100644 --- a/db/queries/project_finder_chunk_candidates.sql +++ b/db/queries/project_finder_chunk_candidates.sql @@ -1,9 +1,14 @@ -WITH terms AS ( - SELECT DISTINCT lower(term) AS term - FROM unnest(%s::text[]) AS term - WHERE length(trim(term)) >= 2 -), -matched AS ( +SELECT source_path, + display_name, + chunk_index, + page_number, + section_title, + category, + quality_score, + chunk_text, + ARRAY[]::text[] AS matched_terms, + 0 AS matched_count +FROM ( SELECT d.source_path, d.display_name, dc.chunk_index, @@ -11,41 +16,19 @@ matched AS ( dc.section_title, dc.category, dc.quality_score, - dc.chunk_text, - array_agg(DISTINCT terms.term ORDER BY terms.term) AS matched_terms, - count(DISTINCT terms.term) AS matched_count + dc.chunk_text FROM document_chunks dc JOIN documents d ON d.id = dc.document_id LEFT JOIN document_pages dp ON dp.id = dc.page_id - JOIN terms ON lower(dc.chunk_text) LIKE '%%' || terms.term || '%%' WHERE d.status_id = 3 + AND lower(dc.chunk_text) ~ %s AND ( lower(dc.chunk_text) ~ '(project|experiment|circuit|build|breadboard|schematic|parts|components|wire|wiring)' OR dc.category IN ('MED_LEVEL_DETAIL', 'TECH_LEVEL_DETAIL', 'CODE_SAMPLE') - OR lower(dc.section_title) LIKE 'code sample%%' + OR lower(coalesce(dc.section_title, '')) LIKE 'code sample%%' ) - GROUP BY d.source_path, - d.display_name, - dc.chunk_index, - dp.page_number, - dc.section_title, - dc.category, - dc.quality_score, - dc.chunk_text -) -SELECT source_path, - display_name, - chunk_index, - page_number, - section_title, - category, - quality_score, - chunk_text, - matched_terms, - matched_count -FROM matched -ORDER BY matched_count DESC, - quality_score DESC, +) AS matched +ORDER BY quality_score DESC, source_path, page_number NULLS LAST, chunk_index diff --git a/tests/test_lab_inventory.py b/tests/test_lab_inventory.py index 6f391b8..8fc0e13 100644 --- a/tests/test_lab_inventory.py +++ b/tests/test_lab_inventory.py @@ -162,6 +162,26 @@ def test_chunk_candidate_uses_inventory_aliases_in_objective(self): self.assertIn("NE555 timer -> LM555", candidate["objective"]) self.assertIn("Breadboard", {part["name"] for part in candidate["missingParts"]}) + def test_chunk_match_annotation_uses_prepared_terms(self): + store = ProjectFinderStore(None, None) + terms = store._prepared_search_terms(["NE555 timer", "10 segment LED", "missing part"]) + rows = [ + { + "chunk_text": "Build an NE-555 timer with a 10-segment LED display.", + "source_path": "book.pdf", + }, + { + "chunk_text": "Copyright and preface text.", + "source_path": "preface.pdf", + }, + ] + + annotated = store._annotate_chunk_matches(rows, terms) + + self.assertEqual(len(annotated), 1) + self.assertEqual(annotated[0]["matched_terms"], ["ne555 timer", "10 segment led"]) + self.assertEqual(annotated[0]["matched_count"], 2) + def test_missing_part_summary_ranks_repeated_gaps(self): store = ProjectFinderStore(None, None) summary = store._missing_part_summary(