From 5ffcc80ebceb3d0a1037f5c56cb19de1799dee05 Mon Sep 17 00:00:00 2001 From: Jan Skowron Date: Fri, 27 Feb 2026 10:05:36 +0100 Subject: [PATCH 1/3] first working version --- .gitignore | 1 + local.sh | 2 +- src/backend/bindings/utils.cpp | 39 +- src/backend/pyproject.toml | 1 + .../search_engine/ltr/build_data/__init__.py | 0 .../ltr/build_data/build_dataset.py | 307 ++++++++++ .../search_engine/ltr/build_data/config.py | 49 ++ .../search_engine/ltr/build_data/features.py | 162 ++++++ .../search_engine/ltr/build_data/io_utils.py | 192 +++++++ .../search_engine/ltr/build_data/test.py | 12 + .../search_engine/query/query_engine.py | 523 +++++++++++++----- .../search_engine/query/query_engine_old.py | 305 ++++++++++ src/backend/search_engine/scoring/bm25.py | 216 ++++++-- src/backend/uv.lock | 11 + 14 files changed, 1639 insertions(+), 181 deletions(-) create mode 100644 src/backend/search_engine/ltr/build_data/__init__.py create mode 100644 src/backend/search_engine/ltr/build_data/build_dataset.py create mode 100644 src/backend/search_engine/ltr/build_data/config.py create mode 100644 src/backend/search_engine/ltr/build_data/features.py create mode 100644 src/backend/search_engine/ltr/build_data/io_utils.py create mode 100644 src/backend/search_engine/ltr/build_data/test.py mode change 100755 => 100644 src/backend/search_engine/query/query_engine.py create mode 100755 src/backend/search_engine/query/query_engine_old.py diff --git a/.gitignore b/.gitignore index d37ac91..ea653ee 100644 --- a/.gitignore +++ b/.gitignore @@ -243,4 +243,5 @@ dist-ssr /src/backend/search_engine/index/bin/ src/backend/search_engine/models/IVFPQ.faiss src/backend/search_engine/models/neuspell-scrnn-probwordnoise +src/backend/search_engine/ltr/build_data/data memory_log.txt diff --git a/local.sh b/local.sh index 58e8bf4..e373085 100755 --- a/local.sh +++ b/local.sh @@ -22,4 +22,4 @@ cleanup() { } trap cleanup EXIT -wait \ No newline at end of file +wait diff --git a/src/backend/bindings/utils.cpp b/src/backend/bindings/utils.cpp index f3091cc..300f562 100644 --- a/src/backend/bindings/utils.cpp +++ b/src/backend/bindings/utils.cpp @@ -316,6 +316,9 @@ class DocStore { std::string get_snippet(uint32_t doc_id, uint64_t tsv_offset); std::optional get(uint32_t doc_id); std::optional get_tsv_offset(uint32_t doc_id); + + std::optional get_title_only(uint32_t doc_id); + uint32_t size() const { return total_docs; } }; @@ -370,6 +373,12 @@ class InvertedIndex { doc_store.open(base_path); } + std::optional get_docfreq(const std::string& term) const { + auto it = term_to_docfreq.find(term); + if (it == term_to_docfreq.end()) return std::nullopt; + return it->second; + } + friend class DocStore; friend class IndexAccessor; }; @@ -731,6 +740,32 @@ std::optional DocStore::get( std::string snippet = get_snippet(doc_id, tsv_offset); return DocInfo{url, title, snippet}; } + +std::optional DocStore::get_title_only(uint32_t doc_id) { + auto it = offsets.find(doc_id); + if (it == offsets.end()) return std::nullopt; + + uint64_t docstore_offset = it->second.docstore_offset; + data_in.clear(); + data_in.seekg(docstore_offset); + + uint32_t url_len; + data_in.read(reinterpret_cast(&url_len), sizeof(url_len)); + if (!data_in) return std::nullopt; + + // skip url bytes + data_in.seekg(static_cast(url_len), std::ios::cur); + + uint32_t title_len; + data_in.read(reinterpret_cast(&title_len), sizeof(title_len)); + if (!data_in) return std::nullopt; + + std::string title(title_len, '\0'); + data_in.read(title.data(), title_len); + if (!data_in) return std::nullopt; + + return ensure_utf8(title); +} // -------------------- std::optional IndexAccessor::get(const std::string& term) { @@ -1010,6 +1045,7 @@ PYBIND11_MODULE(_core, m) { py::class_(m, "DocStore") .def("get", &DocStore::get, py::arg("doc_id")) .def("get_tsv_offset", &DocStore::get_tsv_offset, py::arg("doc_id")) + .def("get_title_only", &DocStore::get_title_only, py::arg("doc_id")) .def_readwrite("query_terms", &DocStore::query_terms); py::class_(m, "IndexAccessor").def("get", &IndexAccessor::get, py::arg("term")); @@ -1019,5 +1055,6 @@ PYBIND11_MODULE(_core, m) { .def_readonly("index", &InvertedIndex::index) .def_readonly("metadata", &InvertedIndex::metadata) .def_readonly("doc_store", &InvertedIndex::doc_store) - .def("clear_cache", &InvertedIndex::clear_cache); + .def("clear_cache", &InvertedIndex::clear_cache) + .def("get_docfreq", &InvertedIndex::get_docfreq, py::arg("term")); } diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 6fe2fc7..d050f28 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "faiss-cpu>=1.13.2", "sentence-transformers>=5.2.3", "psutil>=7.2.2", + "stop-words>=2025.11.4", ] [dependency-groups] diff --git a/src/backend/search_engine/ltr/build_data/__init__.py b/src/backend/search_engine/ltr/build_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/search_engine/ltr/build_data/build_dataset.py b/src/backend/search_engine/ltr/build_data/build_dataset.py new file mode 100644 index 0000000..1217d7d --- /dev/null +++ b/src/backend/search_engine/ltr/build_data/build_dataset.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path + +from tqdm import tqdm + +from .config import DatasetPaths, BuildConfig +from .io_utils import ( + iter_queries, + sample_qids_from_qrels, + load_qrels_for_qids, + load_top100_selection, +) +from .features import parse_query_terms, build_postings_by_term, compute_features_for_doc + +# Your inverted index class from C++ bindings +from cpp_utils import InvertedIndex # type: ignore + + +def split_qids(qids: list[int], *, seed: int) -> tuple[set[int], set[int], set[int]]: + rnd = random.Random(seed) + qids = qids[:] + rnd.shuffle(qids) + + n = len(qids) + n_train = int(0.8 * n) + n_val = int(0.1 * n) + + train = set(qids[:n_train]) + val = set(qids[n_train : n_train + n_val]) + test = set(qids[n_train + n_val :]) + + return train, val, test + + +def pick_negatives( + *, + pos_doc: int, + hard_list: list[int], + soft_candidates: dict[int, int], + hard_n: int, + soft_n: int, + soft_rank: int, + soft_fallback_from: int, + soft_fallback_to: int, +) -> list[int]: + # remove pos from candidates + dedupe while preserving order + hard = [] + seen = {pos_doc} + for d in hard_list: + if d in seen: + continue + hard.append(d) + seen.add(d) + if len(hard) >= hard_n: + break + + # soft: try exact rank first, else fallback from bottom (e.g. 100,99,...,80) + soft: list[int] = [] + if soft_n > 0: + ranks = [] + if soft_rank is not None: + ranks.append(soft_rank) + lo = min(soft_fallback_from, soft_fallback_to) + hi = max(soft_fallback_from, soft_fallback_to) + # go from hi..lo (bottom-up) + ranks.extend(list(range(hi, lo - 1, -1))) + + for r in ranks: + d = soft_candidates.get(r) + if d is None or d in seen: + continue + soft.append(d) + seen.add(d) + if len(soft) >= soft_n: + break + + return hard + soft + + +def build_one_example( + inverted_index, + *, + qid: int, + query: str, + pos_doc: int, + neg_docs: list[int], +) -> dict: + query_terms = parse_query_terms(query) + postings_by_term = build_postings_by_term(inverted_index, query_terms) + + docs = [] + + # positive + fv_pos = compute_features_for_doc( + inverted_index, + doc_id=pos_doc, + query_terms=query_terms, + postings_by_term=postings_by_term, + ) + docs.append( + { + "doc_id": int(pos_doc), + "label": 1, + "features": fv_pos.as_dict(), + } + ) + + # negatives + for d in neg_docs: + fv = compute_features_for_doc( + inverted_index, + doc_id=int(d), + query_terms=query_terms, + postings_by_term=postings_by_term, + ) + docs.append( + { + "doc_id": int(d), + "label": 0, + "features": fv.as_dict(), + } + ) + + return {"qid": int(qid), "query": query, "docs": docs} + + +def write_jsonl(path: Path, rows, *, pretty: bool) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + for r in rows: + if pretty: + f.write(json.dumps(r, ensure_ascii=False)) + else: + f.write(json.dumps(r, ensure_ascii=False, separators=(",", ":"))) + f.write("\n") + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--data-dir", type=str, required=True, help="Folder containing doctrain-*.tsv") + ap.add_argument("--out-dir", type=str, required=True, help="Output folder for train/val/test jsonl") + ap.add_argument("--index-dir", type=str, required=True, help="Path to your built inverted index base dir") + + ap.add_argument("--seed", type=int, default=BuildConfig.seed) + ap.add_argument("--max-queries", type=int, default=100, help="Limit number of qids") + + ap.add_argument("--hard-negatives", type=int, default=BuildConfig.hard_negatives) + ap.add_argument("--soft-negatives", type=int, default=BuildConfig.soft_negatives) + ap.add_argument("--soft-rank", type=int, default=BuildConfig.soft_rank) + ap.add_argument("--soft-fallback-from", type=int, default=BuildConfig.soft_fallback_from) + ap.add_argument("--soft-fallback-to", type=int, default=BuildConfig.soft_fallback_to) + + ap.add_argument("--pretty-json", action="store_true") + + args = ap.parse_args() + + cfg = BuildConfig( + seed=args.seed, + max_queries=None if args.max_queries == -1 else int(args.max_queries), + hard_negatives=int(args.hard_negatives), + soft_negatives=int(args.soft_negatives), + soft_rank=int(args.soft_rank), + soft_fallback_from=int(args.soft_fallback_from), + soft_fallback_to=int(args.soft_fallback_to), + pretty_json=bool(args.pretty_json), + ) + + data_dir = Path(args.data_dir) + out_dir = Path(args.out_dir) + paths = DatasetPaths.from_base(data_dir, out_dir) + + # 1) sample qids from QRELS (guaranteed to have positives) + qid_list = sample_qids_from_qrels(paths.qrels_tsv, seed=cfg.seed, max_queries=cfg.max_queries) + qids = set(qid_list) + + # 1b) build qid->query map ONLY for those qids (streaming scan over queries.tsv) + qid_to_query: dict[int, str] = {} + missing = set(qids) + + for q in iter_queries(paths.queries_tsv): + if q.qid in missing: + qid_to_query[q.qid] = q.query + missing.remove(q.qid) + if not missing: + break + + # drop qids that somehow have no query entry + if missing: + qids = set(qid_to_query.keys()) + qid_list = list(qids) + + # 2) split by qid (avoid leakage across query) + train_qids, val_qids, test_qids = split_qids(qid_list, seed=cfg.seed) + + # 3) load qrels only for selected qids + qrels = load_qrels_for_qids(paths.qrels_tsv, qids) + + # 4) load only needed top100 parts for selected qids + top100_sel = load_top100_selection( + paths.top100_tsv, + qids, + hard_k=cfg.hard_negatives, + soft_rank=cfg.soft_rank, + soft_fallback_from=cfg.soft_fallback_from, + soft_fallback_to=cfg.soft_fallback_to, + ) + + # 5) open index once + inverted_index = InvertedIndex(str(args.index_dir)) + + # 6) streaming build: write train/val/test incrementally (RAM-efficient) + out_dir.mkdir(parents=True, exist_ok=True) + f_train = paths.train_jsonl.open("w", encoding="utf-8") + f_val = paths.val_jsonl.open("w", encoding="utf-8") + f_test = paths.test_jsonl.open("w", encoding="utf-8") + ##########DEBUG + sk_no_qrels = 0 + sk_no_top100 = 0 + sk_no_query = 0 + written = 0 + + try: + for qid in tqdm(qid_list, desc="Building LTR dataset", unit="qid"): + query = qid_to_query.get(qid) + if query is None: + continue + + pos_doc = qrels.get(qid) + if pos_doc is None: + # no judged positive -> skip + continue + + sel = top100_sel.get(qid) + if sel is None: + continue + + neg_docs = pick_negatives( + pos_doc=pos_doc, + hard_list=sel.hard, + soft_candidates=sel.soft_candidates, + hard_n=cfg.hard_negatives, + soft_n=cfg.soft_negatives, + soft_rank=cfg.soft_rank, + soft_fallback_from=cfg.soft_fallback_from, + soft_fallback_to=cfg.soft_fallback_to, + ) + ##########DEBUG + query = qid_to_query.get(qid) + if query is None: + sk_no_query += 1 + continue + + pos_doc = qrels.get(qid) + if pos_doc is None: + sk_no_qrels += 1 + continue + + sel = top100_sel.get(qid) + if sel is None: + sk_no_top100 += 1 + continue + + # ensure final count = 1 + hard + soft + # (If data quality issues cause fewer, we still write what we have.) + row = build_one_example( + inverted_index, + qid=qid, + query=query, + pos_doc=pos_doc, + neg_docs=neg_docs, + ) + #######DEBUG + written += 1 + s = json.dumps(row, ensure_ascii=False, separators=(",", ":")) + if cfg.pretty_json: + s = json.dumps(row, ensure_ascii=False) + + if qid in train_qids: + f_train.write(s + "\n") + elif qid in val_qids: + f_val.write(s + "\n") + else: + f_test.write(s + "\n") + + finally: + f_train.close() + f_val.close() + f_test.close() + + print("SUMMARY") + print(" qids_total:", len(qid_list)) + print(" written:", written) + print(" skipped_no_query:", sk_no_query) + print(" skipped_no_qrels:", sk_no_qrels) + print(" skipped_no_top100:", sk_no_top100) + print(" qrels_loaded:", len(qrels)) + print(" top100_loaded:", len(top100_sel)) + + print(f"Wrote:\n {paths.train_jsonl}\n {paths.val_jsonl}\n {paths.test_jsonl}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/backend/search_engine/ltr/build_data/config.py b/src/backend/search_engine/ltr/build_data/config.py new file mode 100644 index 0000000..67dc976 --- /dev/null +++ b/src/backend/search_engine/ltr/build_data/config.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class DatasetPaths: + queries_tsv: Path + qrels_tsv: Path + top100_tsv: Path + + out_dir: Path + + train_jsonl: Path + val_jsonl: Path + test_jsonl: Path + + @staticmethod + def from_base(data_dir: Path, out_dir: Path) -> "DatasetPaths": + return DatasetPaths( + queries_tsv=data_dir / "msmarco-doctrain-queries.tsv", + qrels_tsv=data_dir / "msmarco-doctrain-qrels.tsv", + top100_tsv=data_dir / "msmarco-doctrain-top100.tsv", + out_dir=out_dir, + train_jsonl=out_dir / "train.jsonl", + val_jsonl=out_dir / "val.jsonl", + test_jsonl=out_dir / "test.jsonl", + ) + + +@dataclass(frozen=True) +class BuildConfig: + seed: int = 13 + + # how many queries (qid) to sample from doctrain-queries + max_queries: int | None = 10 + + # negatives + hard_negatives: int = 10 # take ranks 1..hard_negatives + soft_negatives: int = 1 # take from bottom area / rank=100 by default + + # soft negative strategy + soft_rank: int = 100 # try rank==100 first + soft_fallback_from: int = 80 # if rank==100 conflicts, search ranks 99..80 + soft_fallback_to: int = 100 + + # output format + pretty_json: bool = False \ No newline at end of file diff --git a/src/backend/search_engine/ltr/build_data/features.py b/src/backend/search_engine/ltr/build_data/features.py new file mode 100644 index 0000000..df154df --- /dev/null +++ b/src/backend/search_engine/ltr/build_data/features.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +from typing import Sequence, Mapping + +# your C++ bindings module name might be: cpp_utils or similar. +# In your code snippet: "from cpp_utils import PostingList" +# and normalize_search_query is exported from _core. +# Adjust imports if needed. +from cpp_utils import normalize_search_query, PostingList # type: ignore + +from backend.search_engine.scoring.bm25 import bm25_score_docs_fielded, BM25Config + + +def parse_query_terms(query: str) -> list[str]: + # consistent with your engine (stemming + keep operators) + terms = normalize_search_query(query) + # for LTR features we do NOT want boolean operators as terms + # (they otherwise distort match counts) + drop = {"AND", "OR", "NOT", "&", "|", "-", "(", ")"} + return [t for t in terms if t not in drop] + + +def build_postings_by_term(inverted_index, query_terms: Sequence[str]) -> dict[str, PostingList]: + postings: dict[str, PostingList] = {} + for t in query_terms: + pl = inverted_index.index.get(t) + if pl is None: + continue + postings[t] = pl + return postings + + +def matched_terms_count(doc_id: int, postings_by_term: Mapping[str, PostingList]) -> int: + # how many query terms appear in doc (binary per term) + c = 0 + for t, pl in postings_by_term.items(): + tf = pl.term_frequencies.get(doc_id, 0) + if tf and tf > 0: + c += 1 + return c + + +def phrase_match_indicator(doc_id: int, query_terms: Sequence[str], postings_by_term: Mapping[str, PostingList]) -> int: + """ + Phrase match using positional postings: + For terms t1 t2 ... tn, check existence of positions p, p+1, ..., p+n-1. + We do this via set-intersection shifting positions. + """ + if not query_terms: + return 0 + + # all terms must exist in postings + positions_lists: list[list[int]] = [] + for t in query_terms: + pl = postings_by_term.get(t) + if pl is None: + return 0 + pos = pl.positions.get(doc_id) + if not pos: + return 0 + positions_lists.append([int(x) for x in pos]) + + # fast set-based progressive narrowing + base = set(positions_lists[0]) # candidate start positions of first term + for i in range(1, len(positions_lists)): + shifted = {p - i for p in positions_lists[i]} # positions where phrase could start + base &= shifted + if not base: + return 0 + return 1 + + +@lru_cache(maxsize=200_000) +def _cached_title_terms(inverted_index, doc_id: int) -> tuple[str, ...]: + title = inverted_index.doc_store.get_title_only(int(doc_id)) + if not title: + return tuple() + terms = normalize_search_query(title) + drop = {"AND", "OR", "NOT", "&", "|", "-", "(", ")"} + return tuple(t for t in terms if t not in drop) + + +def in_title_indicator(inverted_index, doc_id: int, query_terms: Sequence[str]) -> int: + title_terms = set(_cached_title_terms(inverted_index, int(doc_id))) + for t in query_terms: + if t in title_terms: + return 1 + return 0 + + +def title_tf(inverted_index, doc_id: int, term: str) -> int: + # used by BM25 scorer; based on cached title terms + return int(_cached_title_terms(inverted_index, int(doc_id)).count(term)) + + +@dataclass(frozen=True) +class FeatureVector: + bm25_body: float + matched_terms: int + matched_frac: float + phrase_match: int + in_title: int + + def as_dict(self) -> dict[str, float | int]: + return { + "bm25_body": self.bm25_body, + "matched_terms": self.matched_terms, + "matched_frac": self.matched_frac, + "phrase_match": self.phrase_match, + "in_title": self.in_title, + } + + +def compute_features_for_doc( + inverted_index, + *, + doc_id: int, + query_terms: Sequence[str], + postings_by_term: Mapping[str, PostingList], +) -> FeatureVector: + # BM25: compute body-only by setting title boost=0 + cfg = BM25Config( + boost_title=0.0, + boost_body=1.0, + b_title=0.0, # irrelevant since boost_title=0 + b_body=BM25Config().b_body, + k1=BM25Config().k1, + idf_threshold=BM25Config().idf_threshold, + clamp_negative_idf=BM25Config().clamp_negative_idf, + min_terms_after_threshold=BM25Config().min_terms_after_threshold, + ) + + scores = bm25_score_docs_fielded( + list(query_terms), + postings_by_term=postings_by_term, + candidate_doc_ids=[int(doc_id)], + num_docs=int(inverted_index.metadata.num_docs), + avg_title_len=float(inverted_index.metadata.avg_title_length), + avg_body_len=float(inverted_index.metadata.avg_body_length), + get_title_len=lambda d: int(inverted_index.metadata.get_title_length(int(d))), + get_body_len=lambda d: int(inverted_index.metadata.get_body_length(int(d))), + get_title_tf=lambda d, t: int(title_tf(inverted_index, int(d), str(t))), + cfg=cfg, + ) + bm25_body = float(scores.get(int(doc_id), 0.0)) + + mt = matched_terms_count(int(doc_id), postings_by_term) + qlen = max(1, len(set(query_terms))) + mf = float(mt) / float(qlen) + + pm = phrase_match_indicator(int(doc_id), list(query_terms), postings_by_term) + it = in_title_indicator(inverted_index, int(doc_id), list(query_terms)) + + return FeatureVector( + bm25_body=bm25_body, + matched_terms=mt, + matched_frac=mf, + phrase_match=pm, + in_title=it, + ) \ No newline at end of file diff --git a/src/backend/search_engine/ltr/build_data/io_utils.py b/src/backend/search_engine/ltr/build_data/io_utils.py new file mode 100644 index 0000000..b594b9f --- /dev/null +++ b/src/backend/search_engine/ltr/build_data/io_utils.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import random +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +@dataclass(frozen=True) +class Query: + qid: int + query: str + + +def iter_queries(path: Path) -> Iterable[Query]: + # doctrain-queries.tsv: qid \t query + with path.open("r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.rstrip("\n") + if not line: + continue + parts = line.split("\t", maxsplit=1) + if len(parts) != 2: + continue + qid_s, q = parts + try: + yield Query(qid=int(qid_s), query=q) + except ValueError: + continue + + +def sample_qids(queries_path: Path, *, seed: int, max_queries: int | None) -> list[Query]: + rnd = random.Random(seed) + + if max_queries is None: + # wenn du wirklich ALLE willst, streamen wir trotzdem, aber ohne shuffle + return list(iter_queries(queries_path)) + + k = int(max_queries) + if k <= 0: + return [] + + reservoir: list[Query] = [] + n = 0 + for q in iter_queries(queries_path): + n += 1 + if len(reservoir) < k: + reservoir.append(q) + else: + j = rnd.randrange(n) + if j < k: + reservoir[j] = q + return reservoir + +def sample_qids_from_qrels(qrels_path: Path, *, seed: int, max_queries: int | None) -> list[int]: + """ + Sample qids from doctrain-qrels.tsv (space-separated). + Uses reservoir sampling if max_queries is set. + """ + rnd = random.Random(seed) + + def iter_qrel_qids() -> Iterable[int]: + with qrels_path.open("r", encoding="utf-8", errors="replace") as f: + for line in f: + parts = line.split() + if len(parts) < 3: + continue + try: + yield int(parts[0]) + except ValueError: + continue + + if max_queries is None: + # all qids (might be large) + return list(iter_qrel_qids()) + + k = int(max_queries) + if k <= 0: + return [] + + reservoir: list[int] = [] + n = 0 + for qid in iter_qrel_qids(): + n += 1 + if len(reservoir) < k: + reservoir.append(qid) + else: + j = rnd.randrange(n) + if j < k: + reservoir[j] = qid + return reservoir + + +def load_qrels_for_qids(qrels_path: Path, qids: set[int]) -> dict[int, int]: + # doctrain-qrels: qid dummy docid dummy (space-separated) + # example: 211691 0 D1499345 1 (label=1) + out: dict[int, int] = {} + with qrels_path.open("r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line: + continue + parts = line.split() + if len(parts) < 3: + continue + try: + qid = int(parts[0]) + except ValueError: + continue + if qid not in qids: + continue + docid = parts[2] + # docid is like "D12345" + if not docid or docid[0] != "D": + continue + try: + out[qid] = int(docid[1:]) + except ValueError: + continue + return out + + +@dataclass +class Top100Selection: + hard: list[int] + soft_candidates: dict[int, int] # rank -> docid + + +def load_top100_selection( + top100_path: Path, + qids: set[int], + *, + hard_k: int, + soft_rank: int, + soft_fallback_from: int, + soft_fallback_to: int, +) -> dict[int, Top100Selection]: + """ + Streaming parse msmarco-doctrain-top100.tsv: + qid \t docid \t rank \t score + We only keep: + - hard ranks: 1..hard_k + - soft ranks: [soft_fallback_from..soft_fallback_to] (so we can pick best available) + """ + soft_low = min(soft_fallback_from, soft_fallback_to) + soft_high = max(soft_fallback_from, soft_fallback_to) + + out: dict[int, Top100Selection] = {} + + with top100_path.open("r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.rstrip("\n") + if not line: + continue + # TREC format: qid Q0 docid rank score run_tag (space-separated) + parts = line.split() + if len(parts) < 4: + continue + try: + qid = int(parts[0]) + except ValueError: + continue + if qid not in qids: + continue + + docid_s = parts[2] + rank_s = parts[3] + + if not docid_s.startswith("D"): + continue + try: + docid = int(docid_s[1:]) + rank = int(rank_s) + except ValueError: + continue + + sel = out.get(qid) + if sel is None: + sel = Top100Selection(hard=[], soft_candidates={}) + out[qid] = sel + + if 1 <= rank <= hard_k: + sel.hard.append(docid) + + if soft_low <= rank <= soft_high: + sel.soft_candidates[rank] = docid + + # small early-stop optimization: if we already got all hard ranks + # and the whole soft bucket for this qid, we could stop per-qid, + # but doing that cleanly is messy; streaming is fast enough. + + return out \ No newline at end of file diff --git a/src/backend/search_engine/ltr/build_data/test.py b/src/backend/search_engine/ltr/build_data/test.py new file mode 100644 index 0000000..5b73683 --- /dev/null +++ b/src/backend/search_engine/ltr/build_data/test.py @@ -0,0 +1,12 @@ +from backend.search_engine.index.index_loader import get_index + +inverted_index = get_index() +docstore = inverted_index.doc_store + +print(docstore.get_title_only(3175109)) + + +""" +uv run --project backend python -m backend.search_engine.ltr.build_data.build_dataset --data-dir /Users/janskowron/VSCode/search-engine/src/backend/search_engine/ltr/build_data/data/msmarco --out-dir /Users/janskowron/VSCode/search-engine/src/backend/search_engine/ltr/build_data/data/ltr_out --index-dir /Users/janskowron/VSCode/search-engine/src/backend/search_engine/index/bin + +""" diff --git a/src/backend/search_engine/query/query_engine.py b/src/backend/search_engine/query/query_engine.py old mode 100755 new mode 100644 index 8f39b96..176b284 --- a/src/backend/search_engine/query/query_engine.py +++ b/src/backend/search_engine/query/query_engine.py @@ -1,9 +1,12 @@ +from __future__ import annotations + import heapq import time +from collections import Counter +from dataclasses import dataclass from cpp_utils import ( # type: ignore [import-untyped] PostingList, - find_docs, normalize_search_query, positional_intersect, ) @@ -19,7 +22,12 @@ Node, QueryTree, ) -from backend.search_engine.scoring.bm25 import BM25Config, bm25_score_docs +from backend.search_engine.scoring.bm25 import ( + BM25Config, + bm25_idf, + bm25_score_docs_fielded, + stop_words, +) from backend.search_engine.semantic_search.query_embeddings import SemanticSearcher from backend.search_engine.spell_correction.spell_correction import repl from backend.search_engine.spell_correction.spell_corrector import get_spell_corrector @@ -27,96 +35,282 @@ logger = get_logger(__name__) +@dataclass(frozen=True) +class RetrievalConfig: + max_terms_for_candidates: int = 3 + max_candidates_total: int = 50_000 + max_candidates_per_term: int = 30_000 + + idf_threshold: float = 0.0 + min_terms_after_threshold: int = 1 + + allow_fallback_full_retrieval: bool = True + + class QueryEngine: def __init__(self, q: str) -> None: self._query = q self.inverted_index = get_index() self.corrector = get_spell_corrector() + + self.retr_cfg = RetrievalConfig( + max_terms_for_candidates=3, + max_candidates_total=10_000, + max_candidates_per_term=10_000, + idf_threshold=0.5, + min_terms_after_threshold=1, + allow_fallback_full_retrieval=False, + ) + + # Fielded BM25 config (title emphasis) + self.bm25_cfg = BM25Config( + k1=1.2, + boost_title=2.5, + boost_body=1.0, + b_title=0.75, + b_body=0.75, + idf_threshold=0.5, + clamp_negative_idf=False, + min_terms_after_threshold=1, + ) + self.semantic_searcher = SemanticSearcher() - def _positional_phrase_search(self, terms: list[str]) -> PostingList: - start = time.perf_counter() - logger.debug(f"Performing phrase search for: {terms}") + # tiny per-query caches (avoid repeated df/idf computation) + self._df_cache: dict[str, int] = {} + self._idf_cache: dict[str, float] = {} - if not terms: - return PostingList(postings=[], term_frequencies={}, positions={}) + @staticmethod + def _empty_pl() -> PostingList: + return PostingList(postings=[], term_frequencies={}, positions={}) - result = self.inverted_index.index.get(terms[0]) + @staticmethod + def _filter_posting_list(pl: PostingList | None, allowed: set[int]) -> PostingList: + if pl is None: + return QueryEngine._empty_pl() - if result is None: - return PostingList(postings=[], term_frequencies={}, positions={}) + filtered_postings = [int(d) for d in pl.postings if int(d) in allowed] - # for each subsequent term, check positions - for i, term in enumerate(terms[1:], start=1): - next_pl = self.inverted_index.index.get(term) + tf_src = ( + pl.term_frequencies + if getattr(pl, "term_frequencies", None) is not None + else {} + ) + filtered_tf = { + int(d): int(tf_src[d]) for d in tf_src.keys() if int(d) in allowed + } + + pos_src = getattr(pl, "positions", None) + if pos_src: + filtered_pos = { + int(d): pos_src[d] for d in pos_src.keys() if int(d) in allowed + } + else: + filtered_pos = {} - if next_pl is None: - return PostingList(postings=[], term_frequencies={}, positions={}) + return PostingList( + postings=filtered_postings, + term_frequencies=filtered_tf, + positions=filtered_pos, + ) - start_positional_intersect = time.perf_counter() - result = positional_intersect(result, next_pl, distance=i) - logger.debug( - f"Positional intersect for term '{term}' " - f"with distance={i} completed in " - f"{time.perf_counter() - start_positional_intersect:.6f}s" - ) - if len(result.postings) == 0: - # warm the cache for remaining terms (needed for snippet generation) - for remaining_term in terms[i + 1 :]: - self.inverted_index.index.get(remaining_term) + @staticmethod + def _filter_posting_list_postings_only( + pl: PostingList | None, allowed: set[int] + ) -> PostingList: + if pl is None: + return QueryEngine._empty_pl() + filtered_postings = [int(d) for d in pl.postings if int(d) in allowed] + return PostingList( + postings=filtered_postings, term_frequencies={}, positions={} + ) + + @staticmethod + def _bool_op_postings( + left: PostingList, right: PostingList, op: str + ) -> PostingList: + lset = set(int(d) for d in left.postings) + rset = set(int(d) for d in right.postings) + + if op == "AND": + out = lset & rset + elif op == "OR": + out = lset | rset + elif op == "NOT": + out = lset - rset + else: + out = set() + + return PostingList(postings=sorted(out), term_frequencies={}, positions={}) + + # DF / IDF caching + def _df_for_term(self, term: str) -> int: + cached = self._df_cache.get(term) + if cached is not None: + return cached + + df = self.inverted_index.get_docfreq(term) + df_i = int(df) if df is not None else 0 + + self._df_cache[term] = df_i + return df_i + + def _idf_for_term(self, term: str) -> float: + cached = self._idf_cache.get(term) + if cached is not None: + return cached + + df = self._df_for_term(term) + if df <= 0: + self._idf_cache[term] = 0.0 + return 0.0 + + md = self.inverted_index.metadata + idf = bm25_idf( + int(md.num_docs), int(df), clamp_negative=self.bm25_cfg.clamp_negative_idf + ) + self._idf_cache[term] = float(idf) + return float(idf) + + # candidate selection + def _select_terms_for_candidates(self, query_terms: list[str]) -> list[str]: + term_idf: list[tuple[str, float]] = [] + for t in query_terms: + if t in AND or t in OR or t in NOT: + continue + idf = self._idf_for_term(t) + if idf > 0.0 or self.retr_cfg.idf_threshold <= 0.0: + term_idf.append((t, idf)) + + term_idf.sort(key=lambda x: x[1], reverse=True) + + kept = [t for (t, idf) in term_idf if idf >= self.retr_cfg.idf_threshold] + if len(kept) < max(1, int(self.retr_cfg.min_terms_after_threshold)): + kept = [ + t + for (t, _) in term_idf[ + : max(1, int(self.retr_cfg.min_terms_after_threshold)) + ] + ] + + return kept[: max(1, int(self.retr_cfg.max_terms_for_candidates))] + + def _build_candidates_from_terms(self, terms: list[str]) -> list[int]: + start = time.perf_counter() + + cand: set[int] = set() + total_cap = int(self.retr_cfg.max_candidates_total) + per_term_cap = int(self.retr_cfg.max_candidates_per_term) + + for t in terms: + pl = self.inverted_index.index.get(t) + if pl is None: + continue + + for d in pl.postings[:per_term_cap]: + cand.add(int(d)) + if len(cand) >= total_cap: + break + if len(cand) >= total_cap: break - end = time.perf_counter() logger.debug( - f"Result docs: {len(result.postings)}, " - f"Execution time: {end - start:.6f} seconds" + f"Candidate generation using terms={terms} -> {len(cand)} candidates " + f"in {time.perf_counter() - start:.6f}s" ) + return list(cand) - return result - - def _bool_search(self, node: Node | None) -> PostingList: + def _bool_search( + self, node: Node | None, allowed: set[int], cand_list: list[int] + ) -> PostingList: start = time.perf_counter() logger.debug(f"Evaluating node: {getattr(node, 'value', None)}") if node is None: - return PostingList(postings=[], term_frequencies={}, positions={}) + return self._empty_pl() - if node.value not in AND | OR | NOT: + if node.value not in (AND | OR | NOT): pl = self.inverted_index.index.get(node.value) - result = pl or PostingList(postings=[], term_frequencies={}, positions={}) + if pl is None: + result = self._empty_pl() + else: + tf_map = pl.term_frequencies # dict-like: doc_id -> tf + out = [d for d in cand_list if d in tf_map] + result = PostingList(postings=out, term_frequencies={}, positions={}) elif node.value in AND: - # check if one of the nodes has NOT child left_is_not = node.left and node.left.value in NOT right_is_not = node.right and node.right.value in NOT if left_is_not: - # NOT A AND B -> B minus A - not_docs = self._bool_search(node.left.right if node.left else None) - right = self._bool_search(node.right) - result = find_docs(right, not_docs, "NOT") + not_docs = self._bool_search( + node.left.right if node.left else None, allowed, cand_list + ) + right = self._bool_search(node.right, allowed, cand_list) + result = self._bool_op_postings(right, not_docs, "NOT") elif right_is_not: - # A AND NOT B -> A minus B - left = self._bool_search(node.left) - not_docs = self._bool_search(node.right.right if node.right else None) - result = find_docs(left, not_docs, "NOT") + left = self._bool_search(node.left, allowed, cand_list) + not_docs = self._bool_search( + node.right.right if node.right else None, allowed, cand_list + ) + result = self._bool_op_postings(left, not_docs, "NOT") else: - # regular AND without NOT - left = self._bool_search(node.left) - right = self._bool_search(node.right) - result = find_docs(left, right, "AND") + left = self._bool_search(node.left, allowed, cand_list) + right = self._bool_search(node.right, allowed, cand_list) + result = self._bool_op_postings(left, right, "AND") else: # OR - left = self._bool_search(node.left) - right = self._bool_search(node.right) - result = find_docs(left, right, "OR") + left = self._bool_search(node.left, allowed, cand_list) + right = self._bool_search(node.right, allowed, cand_list) + result = self._bool_op_postings(left, right, "OR") - end = time.perf_counter() logger.debug( f"Node={node.value!r}, Result docs={len(result.postings)}, " - f"Execution time: {end - start:.6f} seconds" + f"Execution time: {time.perf_counter() - start:.6f} seconds" + ) + return result + + def _positional_phrase_search( + self, terms: list[str], allowed: set[int] + ) -> PostingList: + start = time.perf_counter() + logger.debug(f"Performing phrase search for: {terms}") + + if not terms: + return self._empty_pl() + + first = self._filter_posting_list( + self.inverted_index.index.get(terms[0]), allowed + ) + if len(first.postings) == 0: + return self._empty_pl() + + result = first + for i, term in enumerate(terms[1:], start=1): + next_pl = self._filter_posting_list( + self.inverted_index.index.get(term), allowed + ) + if len(next_pl.postings) == 0: + return self._empty_pl() + + start_positional_intersect = time.perf_counter() + result = positional_intersect(result, next_pl, distance=i) + logger.debug( + f"Positional intersect for term '{term}' " + f"with distance={i} completed in " + f"{time.perf_counter() - start_positional_intersect:.6f}s" + ) + if len(result.postings) == 0: + # warm the cache for remaining terms (needed for snippet generation) + for remaining_term in terms[i + 1 :]: + self.inverted_index.index.get(remaining_term) + break + + logger.debug( + f"Result docs: {len(result.postings)}, " + f"Execution time: {time.perf_counter() - start:.6f} seconds" ) return result @@ -124,12 +318,9 @@ def _bool_search(self, node: Node | None) -> PostingList: def _to_boolean_normalized_query(tokens: list[str]) -> list[str]: if not tokens: return [] - query_str = tokens[0] - for term in tokens[1:]: query_str = f"({query_str} AND {term})" - return normalize_search_query(query_str) @staticmethod @@ -152,105 +343,177 @@ def _reciprocal_rank_fusion( def search_results(self, limit: int = 10) -> SearchResults: start = time.perf_counter() - logger.debug("Starting query execution") + logger.debug("Starting query execution...") qt = QueryTree() + + t_norm = time.perf_counter() normalized_tokens = normalize_search_query(self._query) + logger.debug( + f"normalize_search_query time: {time.perf_counter() - t_norm:.6f}s" + ) logger.debug(f"Normalized search query: {normalized_tokens}") raw_query = self._query.strip() + t_corr = time.perf_counter() correction = repl(self.corrector, raw_query) + logger.debug( + f"spell correction total time: {time.perf_counter() - t_corr:.6f}s" + ) - if not qt._has_operators(normalized_tokens): - self.inverted_index.doc_store.query_terms = list( - set(normalized_tokens) - ) # needed for snippets - if (raw_query.startswith('"') and raw_query.endswith('"')) or ( - raw_query.startswith("'") and raw_query.endswith("'") - ): - # positional phrase search - logger.debug("Executing positional phrase query search...") - normalized_tokens_no_quots = normalize_search_query(raw_query[1:-1]) - result = self._positional_phrase_search(normalized_tokens_no_quots) - else: - # any order -> create AND query - logger.debug("Executing phrase query search...") - and_query = QueryEngine._to_boolean_normalized_query(normalized_tokens) - logger.debug(f"Converted to AND query: {and_query}") - qt.parse_query(and_query) - logger.debug(f"Query tree: {qt.root}") - result = self._bool_search(qt.root) - else: - logger.debug("Executing bool query search...") + base_terms = [t for t in normalized_tokens if t not in (AND | OR | NOT)] + + t_set_qt = time.perf_counter() + self.inverted_index.doc_store.query_terms = list(set(base_terms)) + logger.debug( + f"set doc_store.query_terms (base) time: {time.perf_counter() - t_set_qt:.6f}s" + ) + + t_flags = time.perf_counter() + is_quoted_phrase = (raw_query.startswith('"') and raw_query.endswith('"')) or ( + raw_query.startswith("'") and raw_query.endswith("'") + ) + logger.debug(f"quoted phrase check time: {time.perf_counter() - t_flags:.6f}s") + + t_has_ops = time.perf_counter() + has_ops = qt._has_operators(normalized_tokens) + logger.debug(f"_has_operators time: {time.perf_counter() - t_has_ops:.6f}s") + + if has_ops: try: + t_parse_ops = time.perf_counter() qt.parse_query(normalized_tokens) - self.inverted_index.doc_store.query_terms = ( - qt.unique_terms - ) # needed for snippets - logger.debug(f"Query tree: {qt.root}") - result = self._bool_search(qt.root) + logger.debug( + f"parse_query (ops) time: {time.perf_counter() - t_parse_ops:.6f}s" + ) + + t_set_qt2 = time.perf_counter() + self.inverted_index.doc_store.query_terms = qt.unique_terms + logger.debug( + f"set doc_store.query_terms (ops) time: {time.perf_counter() - t_set_qt2:.6f}s" + ) except InvalidOperatorError as e: logger.error(f"Invalid query syntax: {e}") raise - has_boolean_results = result is not None and len(result.postings) > 0 + _query_terms: list[str] = ( + list(qt.unique_terms) + if (has_ops and getattr(qt, "unique_terms", None)) + else list(dict.fromkeys(base_terms)) + ) - if has_boolean_results: + query_terms = [term for term in _query_terms if term not in stop_words] + cand_terms = self._select_terms_for_candidates(query_terms) + candidate_doc_ids = self._build_candidates_from_terms(cand_terms) + cand_set = set(candidate_doc_ids) + + logger.debug(f"candidate_doc_ids count={len(candidate_doc_ids)}") + + t_filter = time.perf_counter() + restricted_result: PostingList + + if is_quoted_phrase: + logger.debug("Executing positional phrase query search...") + phrase_terms = normalize_search_query(raw_query[1:-1]) + restricted_result = self._positional_phrase_search(phrase_terms, cand_set) + + elif has_ops: + logger.debug("Executing bool query search...") + logger.debug(f"Query tree: {qt.root}") + restricted_result = self._bool_search(qt.root, cand_set, candidate_doc_ids) + + else: + logger.debug("Executing phrase query search...") + and_query = self._to_boolean_normalized_query(query_terms) + + logger.debug(f"Converted to AND query: {and_query}") + + qt2 = QueryTree() + t_parse_no_ops = time.perf_counter() + qt2.parse_query(and_query) logger.debug( - f"Found {len(result.postings)} results in {time.perf_counter() - start:.6f} seconds" + f"parse_query (no ops) time: {time.perf_counter() - t_parse_no_ops:.6f}s" ) + logger.debug(f"Query tree: {qt2.root}") - # candidates: bool/phrase search returns doc_ids in result.postings - candidate_doc_ids = list(result.postings) - metadata = self.inverted_index.metadata + t_bool = time.perf_counter() + restricted_result = self._bool_search(qt2.root, cand_set, candidate_doc_ids) + logger.debug(f"Bool search time: {time.perf_counter() - t_bool:.6f}s") - # score which query terms - # - bool queries: qt.unique_terms - # - AND-auto-query w/o operators: normalized_tokens - query_terms: list[str] = ( - list(qt.unique_terms) - if getattr(qt, "unique_terms", None) - else list(dict.fromkeys(normalized_tokens)) + logger.debug(f"Filter time: {time.perf_counter() - t_filter:.6f}s") + + has_boolean_results = ( + restricted_result is not None and len(restricted_result.postings) > 0 + ) + + # scoring (Fielded BM25) + if has_boolean_results: + logger.debug( + f"Found {len(restricted_result.postings)} results " + f"in {time.perf_counter() - start:.6f} seconds" ) + t_score = time.perf_counter() + metadata = self.inverted_index.metadata - t_bm25 = time.perf_counter() + t_final_ids = time.perf_counter() + final_candidate_doc_ids = list(restricted_result.postings) + logger.debug( + f"final_candidate_doc_ids build time: {time.perf_counter() - t_final_ids:.6f}s" + ) - bm25_scores = bm25_score_docs( + # title TF cache: doc_id -> Counter(term->tf) for only query terms. + # IMPORTANT: uses get_title_only (no snippet IO). + qterm_set = set(query_terms) + title_tf_cache: dict[int, Counter[str]] = {} + + def get_title_tf(doc_id: int, term: str) -> int: + c = title_tf_cache.get(doc_id) + if c is None: + title_opt = self.inverted_index.doc_store.get_title_only( + int(doc_id) + ) + title = title_opt or "" + toks = normalize_search_query(title) # stem+tokenize in C++ + c = Counter(t for t in toks if t in qterm_set) + title_tf_cache[doc_id] = c + return int(c.get(term, 0)) + + scores = bm25_score_docs_fielded( query_terms=query_terms, postings_by_term=self.inverted_index.index, # term -> PostingList - candidate_doc_ids=candidate_doc_ids, - num_docs=metadata.num_docs, - avgdl=metadata.avg_body_length, - get_doc_length=metadata.get_doc_length, - cfg=BM25Config( - k1=1.2, b=0.75, idf_threshold=0.0, clamp_negative_idf=True - ), + candidate_doc_ids=final_candidate_doc_ids, + num_docs=int(metadata.num_docs), + avg_title_len=float(metadata.avg_title_length), + avg_body_len=float(metadata.avg_body_length), + get_title_len=metadata.get_title_length, + get_body_len=metadata.get_body_length, + get_title_tf=get_title_tf, + cfg=self.bm25_cfg, + ) + logger.debug( + f"Fielded BM25 scoring time: {time.perf_counter() - t_score:.6f}s" ) - logger.debug(f"BM25 scoring time: {time.perf_counter() - t_bm25:.6f}s") - - t_sort_bm25 = time.perf_counter() - - # sort doc_ids acc to score + # top-k BM25 + t_sort = time.perf_counter() bm25_ranked_top = heapq.nlargest( limit, ( - (doc_id, bm25_scores.get(doc_id, 0.0)) - for doc_id in candidate_doc_ids + (doc_id, scores.get(int(doc_id), 0.0)) + for doc_id in final_candidate_doc_ids ), key=lambda x: x[1], ) - - logger.debug(f"Ranking sort time: {time.perf_counter() - t_sort_bm25:.6f}s") + logger.debug(f"Ranking sort time: {time.perf_counter() - t_sort:.6f}s") else: logger.debug( "No boolean results found, falling back to semantic search only" ) bm25_ranked_top = [] + # semantic search t_semantic = time.perf_counter() - semantic_scores = self.semantic_searcher.search(raw_query, limit * 2) semantic_ranked_top = heapq.nlargest( limit, @@ -259,18 +522,20 @@ def search_results(self, limit: int = 10) -> SearchResults: ) logger.debug(f"Semantic ranking time: {time.perf_counter() - t_semantic:.6f}s") + # RRF ranked_lists = [semantic_ranked_top] if bm25_ranked_top: ranked_lists.append(bm25_ranked_top) - t_rrf = time.perf_counter() final_top = QueryEngine._reciprocal_rank_fusion( lists=ranked_lists, top_n=limit, k=60 ) - logger.debug(f"RRF fusion time: {time.perf_counter() - t_rrf:.6f}s") - search_results = [] + t_top = time.perf_counter() + search_results: list[SearchResult] = [] + for doc_id, rrf_score in final_top: + doc_id = int(doc_id) doc_data = self.inverted_index.doc_store.get(doc_id) if doc_data is None: continue @@ -278,21 +543,26 @@ def search_results(self, limit: int = 10) -> SearchResults: url = doc_data.url if url is None: continue + title = doc_data.title or "Untitled" snippet = doc_data.snippet try: - search_result = SearchResult( - document_id=doc_id, - url=url, # type: ignore[arg-type] - title=title, - snippet=snippet, - rrf_score=rrf_score, + search_results.append( + SearchResult( + document_id=doc_id, + url=url, # type: ignore[arg-type] + title=title, + snippet=snippet, + rrf_score=rrf_score, + ) ) - search_results.append(search_result) except Exception as e: logger.error(f"Error creating SearchResult for doc_id {doc_id}: {e}") - continue + + logger.debug( + f"Build top-{limit} results time: {time.perf_counter() - t_top:.6f}s" + ) end = time.perf_counter() logger.debug( @@ -301,5 +571,4 @@ def search_results(self, limit: int = 10) -> SearchResults: ) # clear cache to free memory self.inverted_index.clear_cache() - return SearchResults(search_results=search_results, correction=correction) diff --git a/src/backend/search_engine/query/query_engine_old.py b/src/backend/search_engine/query/query_engine_old.py new file mode 100755 index 0000000..8f39b96 --- /dev/null +++ b/src/backend/search_engine/query/query_engine_old.py @@ -0,0 +1,305 @@ +import heapq +import time + +from cpp_utils import ( # type: ignore [import-untyped] + PostingList, + find_docs, + normalize_search_query, + positional_intersect, +) + +from backend.logging_config import get_logger +from backend.search_engine.error_handling import InvalidOperatorError +from backend.search_engine.index.index_loader import get_index +from backend.search_engine.models.index import SearchResult, SearchResults +from backend.search_engine.query.query_preprocessing import ( + AND, + NOT, + OR, + Node, + QueryTree, +) +from backend.search_engine.scoring.bm25 import BM25Config, bm25_score_docs +from backend.search_engine.semantic_search.query_embeddings import SemanticSearcher +from backend.search_engine.spell_correction.spell_correction import repl +from backend.search_engine.spell_correction.spell_corrector import get_spell_corrector + +logger = get_logger(__name__) + + +class QueryEngine: + def __init__(self, q: str) -> None: + self._query = q + self.inverted_index = get_index() + self.corrector = get_spell_corrector() + self.semantic_searcher = SemanticSearcher() + + def _positional_phrase_search(self, terms: list[str]) -> PostingList: + start = time.perf_counter() + logger.debug(f"Performing phrase search for: {terms}") + + if not terms: + return PostingList(postings=[], term_frequencies={}, positions={}) + + result = self.inverted_index.index.get(terms[0]) + + if result is None: + return PostingList(postings=[], term_frequencies={}, positions={}) + + # for each subsequent term, check positions + for i, term in enumerate(terms[1:], start=1): + next_pl = self.inverted_index.index.get(term) + + if next_pl is None: + return PostingList(postings=[], term_frequencies={}, positions={}) + + start_positional_intersect = time.perf_counter() + result = positional_intersect(result, next_pl, distance=i) + logger.debug( + f"Positional intersect for term '{term}' " + f"with distance={i} completed in " + f"{time.perf_counter() - start_positional_intersect:.6f}s" + ) + if len(result.postings) == 0: + # warm the cache for remaining terms (needed for snippet generation) + for remaining_term in terms[i + 1 :]: + self.inverted_index.index.get(remaining_term) + break + + end = time.perf_counter() + logger.debug( + f"Result docs: {len(result.postings)}, " + f"Execution time: {end - start:.6f} seconds" + ) + + return result + + def _bool_search(self, node: Node | None) -> PostingList: + start = time.perf_counter() + logger.debug(f"Evaluating node: {getattr(node, 'value', None)}") + + if node is None: + return PostingList(postings=[], term_frequencies={}, positions={}) + + if node.value not in AND | OR | NOT: + pl = self.inverted_index.index.get(node.value) + result = pl or PostingList(postings=[], term_frequencies={}, positions={}) + + elif node.value in AND: + # check if one of the nodes has NOT child + left_is_not = node.left and node.left.value in NOT + right_is_not = node.right and node.right.value in NOT + + if left_is_not: + # NOT A AND B -> B minus A + not_docs = self._bool_search(node.left.right if node.left else None) + right = self._bool_search(node.right) + result = find_docs(right, not_docs, "NOT") + + elif right_is_not: + # A AND NOT B -> A minus B + left = self._bool_search(node.left) + not_docs = self._bool_search(node.right.right if node.right else None) + result = find_docs(left, not_docs, "NOT") + + else: + # regular AND without NOT + left = self._bool_search(node.left) + right = self._bool_search(node.right) + result = find_docs(left, right, "AND") + + else: # OR + left = self._bool_search(node.left) + right = self._bool_search(node.right) + result = find_docs(left, right, "OR") + + end = time.perf_counter() + logger.debug( + f"Node={node.value!r}, Result docs={len(result.postings)}, " + f"Execution time: {end - start:.6f} seconds" + ) + return result + + @staticmethod + def _to_boolean_normalized_query(tokens: list[str]) -> list[str]: + if not tokens: + return [] + + query_str = tokens[0] + + for term in tokens[1:]: + query_str = f"({query_str} AND {term})" + + return normalize_search_query(query_str) + + @staticmethod + def _reciprocal_rank_fusion( + lists: list[list[tuple[int, float]]], top_n: int, k: int + ) -> list[tuple[int, float]]: + """ + Fuse multiple ranked lists using Reciprocal Rank Fusion (RRF). + + Returns: + List of top_n (doc_id, combined_score) sorted by RRF score + """ + rrf_scores: dict[int, float] = {} + + for ranked_list in lists: + for rank, (doc_id, _) in enumerate(ranked_list): + rrf_scores[doc_id] = rrf_scores.get(doc_id, 0.0) + 1 / (k + rank + 1) + + return heapq.nlargest(top_n, rrf_scores.items(), key=lambda x: x[1]) + + def search_results(self, limit: int = 10) -> SearchResults: + start = time.perf_counter() + logger.debug("Starting query execution") + + qt = QueryTree() + normalized_tokens = normalize_search_query(self._query) + logger.debug(f"Normalized search query: {normalized_tokens}") + + raw_query = self._query.strip() + + correction = repl(self.corrector, raw_query) + + if not qt._has_operators(normalized_tokens): + self.inverted_index.doc_store.query_terms = list( + set(normalized_tokens) + ) # needed for snippets + if (raw_query.startswith('"') and raw_query.endswith('"')) or ( + raw_query.startswith("'") and raw_query.endswith("'") + ): + # positional phrase search + logger.debug("Executing positional phrase query search...") + normalized_tokens_no_quots = normalize_search_query(raw_query[1:-1]) + result = self._positional_phrase_search(normalized_tokens_no_quots) + else: + # any order -> create AND query + logger.debug("Executing phrase query search...") + and_query = QueryEngine._to_boolean_normalized_query(normalized_tokens) + logger.debug(f"Converted to AND query: {and_query}") + qt.parse_query(and_query) + logger.debug(f"Query tree: {qt.root}") + result = self._bool_search(qt.root) + else: + logger.debug("Executing bool query search...") + try: + qt.parse_query(normalized_tokens) + self.inverted_index.doc_store.query_terms = ( + qt.unique_terms + ) # needed for snippets + logger.debug(f"Query tree: {qt.root}") + result = self._bool_search(qt.root) + except InvalidOperatorError as e: + logger.error(f"Invalid query syntax: {e}") + raise + + has_boolean_results = result is not None and len(result.postings) > 0 + + if has_boolean_results: + logger.debug( + f"Found {len(result.postings)} results in {time.perf_counter() - start:.6f} seconds" + ) + + # candidates: bool/phrase search returns doc_ids in result.postings + candidate_doc_ids = list(result.postings) + metadata = self.inverted_index.metadata + + # score which query terms + # - bool queries: qt.unique_terms + # - AND-auto-query w/o operators: normalized_tokens + query_terms: list[str] = ( + list(qt.unique_terms) + if getattr(qt, "unique_terms", None) + else list(dict.fromkeys(normalized_tokens)) + ) + + t_bm25 = time.perf_counter() + + bm25_scores = bm25_score_docs( + query_terms=query_terms, + postings_by_term=self.inverted_index.index, # term -> PostingList + candidate_doc_ids=candidate_doc_ids, + num_docs=metadata.num_docs, + avgdl=metadata.avg_body_length, + get_doc_length=metadata.get_doc_length, + cfg=BM25Config( + k1=1.2, b=0.75, idf_threshold=0.0, clamp_negative_idf=True + ), + ) + + logger.debug(f"BM25 scoring time: {time.perf_counter() - t_bm25:.6f}s") + + t_sort_bm25 = time.perf_counter() + + # sort doc_ids acc to score + bm25_ranked_top = heapq.nlargest( + limit, + ( + (doc_id, bm25_scores.get(doc_id, 0.0)) + for doc_id in candidate_doc_ids + ), + key=lambda x: x[1], + ) + + logger.debug(f"Ranking sort time: {time.perf_counter() - t_sort_bm25:.6f}s") + else: + logger.debug( + "No boolean results found, falling back to semantic search only" + ) + bm25_ranked_top = [] + + t_semantic = time.perf_counter() + + semantic_scores = self.semantic_searcher.search(raw_query, limit * 2) + semantic_ranked_top = heapq.nlargest( + limit, + semantic_scores, + key=lambda x: x[1], + ) + logger.debug(f"Semantic ranking time: {time.perf_counter() - t_semantic:.6f}s") + + ranked_lists = [semantic_ranked_top] + if bm25_ranked_top: + ranked_lists.append(bm25_ranked_top) + + t_rrf = time.perf_counter() + final_top = QueryEngine._reciprocal_rank_fusion( + lists=ranked_lists, top_n=limit, k=60 + ) + logger.debug(f"RRF fusion time: {time.perf_counter() - t_rrf:.6f}s") + + search_results = [] + for doc_id, rrf_score in final_top: + doc_data = self.inverted_index.doc_store.get(doc_id) + if doc_data is None: + continue + + url = doc_data.url + if url is None: + continue + title = doc_data.title or "Untitled" + snippet = doc_data.snippet + + try: + search_result = SearchResult( + document_id=doc_id, + url=url, # type: ignore[arg-type] + title=title, + snippet=snippet, + rrf_score=rrf_score, + ) + search_results.append(search_result) + except Exception as e: + logger.error(f"Error creating SearchResult for doc_id {doc_id}: {e}") + continue + + end = time.perf_counter() + logger.debug( + f"Returned {len(search_results)} results. " + f"Total execution time: {end - start:.6f} seconds" + ) + # clear cache to free memory + self.inverted_index.clear_cache() + + return SearchResults(search_results=search_results, correction=correction) diff --git a/src/backend/search_engine/scoring/bm25.py b/src/backend/search_engine/scoring/bm25.py index 398b761..2230547 100644 --- a/src/backend/search_engine/scoring/bm25.py +++ b/src/backend/search_engine/scoring/bm25.py @@ -1,24 +1,107 @@ -# backend/search_engine/scoring/bm25.py from __future__ import annotations import math +from collections.abc import Callable from dataclasses import dataclass from typing import Iterable, Mapping, Sequence -from collections.abc import Callable from cpp_utils import PostingList # type: ignore [import-untyped] +stop_words = { + "the", + "and", + "to", + "of", + "a", + "in", + "is", + "it", + "you", + "that", + "he", + "was", + "for", + "on", + "are", + "with", + "as", + "i", + "his", + "they", + "be", + "at", + "one", + "have", + "this", + "from", + "or", + "had", + "by", + "but", + "not", + "what", + "all", + "were", + "we", + "when", + "your", + "can", + "said", + "there", + "use", + "an", + "each", + "which", + "do", + "how", + "their", + "if", + "will", + "up", + "other", + "about", + "out", + "many", + "then", + "them", + "these", + "so", + "some", + "her", + "would", + "make", + "like", + "him", + "into", + "time", + "has", + "look", + "two", + "more", + "write", + "go", + "see", +} + @dataclass(frozen=True) class BM25Config: - k1: float = 1.2 # how strong tf influence is - b: float = 0.75 # level of document normalization + # global saturation parameter + k1: float = 1.2 + + # Fielded BM25 boosts (title emphasis) + boost_title: float = 2.5 + boost_body: float = 1.0 + + # per-field length normalization strengths + b_title: float = 0.75 + b_body: float = 0.75 # ignore terms with idf < idf_threshold - idf_threshold: float = 0.0 + idf_threshold: float = 0.5 # clamp negative idf to 0 clamp_negative_idf: bool = True - + # keep at least n terms after thresholding min_terms_after_threshold: int = 1 @@ -26,51 +109,51 @@ def bm25_idf(num_docs: int, df: int, *, clamp_negative: bool = True) -> float: """ idf: ln((N - df + 0.5) / (df + 0.5)) """ - if num_docs <= 0: + if num_docs <= 0 or df <= 0: return 0.0 - if df <= 0: - return 0.0 - val = math.log((num_docs - df + 0.5) / (df + 0.5)) if clamp_negative and val < 0.0: return 0.0 return val -def bm25_term_contribution( - tf: int, - doc_len: int, - avgdl: float, - *, - idf: float, - k1: float, - b: float, +def _field_norm_tf( + tf: int, *, field_len: int, avg_field_len: float, b_f: float ) -> float: + """ + Field-normalized TF: + tf / (1 - b_f + b_f * len_f(d) / avglen_f) + """ if tf <= 0: return 0.0 - if avgdl <= 0: - avgdl = 1.0 + if avg_field_len <= 0.0: + avg_field_len = 1.0 + if field_len <= 0: + field_len = 1 - denom = tf + k1 * (1.0 - b + b * (doc_len / avgdl)) - return idf * (tf * (k1 + 1.0) / denom) + denom = (1.0 - b_f) + b_f * (float(field_len) / float(avg_field_len)) + if denom <= 0.0: + return float(tf) + return float(tf) / denom -def bm25_score_docs( +def bm25_score_docs_fielded( query_terms: Sequence[str], *, postings_by_term: Mapping[str, PostingList], candidate_doc_ids: Iterable[int], num_docs: int, - avgdl: float, - get_doc_length: Callable[[int], int], + avg_title_len: float, + avg_body_len: float, + get_title_len: Callable[[int], int], + get_body_len: Callable[[int], int], + get_title_tf: Callable[[int, str], int], cfg: BM25Config = BM25Config(), ) -> dict[int, float]: - # materialize candidates once - cand_list = list(candidate_doc_ids) - cand_set = set(cand_list) + cand_list = [int(d) for d in candidate_doc_ids] - # compute idf for al query terms first - term_idf_all: list[tuple[str, float]] = [] # (term, idf) + # --- idf compute once --- + term_idf_all: list[tuple[str, float]] = [] for t in query_terms: pl = postings_by_term.get(t) if pl is None: @@ -78,46 +161,75 @@ def bm25_score_docs( df = getattr(pl, "doc_frequency", None) if df is None: df = len(pl.postings) + term_idf_all.append( + ( + t, + float( + bm25_idf(num_docs, int(df), clamp_negative=cfg.clamp_negative_idf) + ), + ) + ) - idf = bm25_idf(num_docs, int(df), clamp_negative=cfg.clamp_negative_idf) - term_idf_all.append((t, idf)) - - # order by decreasing idf term_idf_all.sort(key=lambda x: x[1], reverse=True) - # thresholding term_idf: dict[str, float] = { t: idf for (t, idf) in term_idf_all if idf >= cfg.idf_threshold } - - # ensure keep at least n best terms min_keep = max(1, int(cfg.min_terms_after_threshold)) if len(term_idf) < min_keep: term_idf = {t: idf for (t, idf) in term_idf_all[:min_keep]} - # scoring only docs that actually appear in term postings (tf>0) - scores: dict[int, float] = {int(d): 0.0 for d in cand_list} + scores: dict[int, float] = {d: 0.0 for d in cand_list} + + # --- length caches --- + title_len_cache: dict[int, int] = {} + body_len_cache: dict[int, int] = {} + + def _tlen(d: int) -> int: + v = title_len_cache.get(d) + if v is None: + v = int(get_title_len(d)) + title_len_cache[d] = v + return v + + def _blen(d: int) -> int: + v = body_len_cache.get(d) + if v is None: + v = int(get_body_len(d)) + body_len_cache[d] = v + return v + k1 = float(cfg.k1) + k1p1 = k1 + 1.0 + + # --- iterate candidates (FAST) --- for t, idf in term_idf.items(): pl = postings_by_term.get(t) if pl is None: continue - tf_map = dict(pl.term_frequencies) - - for doc_id, tf in tf_map.items(): - doc_id = int(doc_id) - if doc_id not in cand_set: + tf_body_map = pl.term_frequencies # python mapping already + for d in cand_list: + tf_body = int(tf_body_map.get(d, 0)) + if tf_body <= 0: + # candidates are generated from body postings, so in practice tf_body>0 + # for the generating term; for OR-queries it may be 0 -> skip continue - dl = int(get_doc_length(doc_id)) - scores[doc_id] += bm25_term_contribution( - tf=int(tf), - doc_len=dl, - avgdl=avgdl, - idf=idf, - k1=cfg.k1, - b=cfg.b, + # title tf only computed for docs we actually score + tf_title = int(get_title_tf(d, t)) + + tf_norm = cfg.boost_title * _field_norm_tf( + tf_title, + field_len=_tlen(d), + avg_field_len=avg_title_len, + b_f=cfg.b_title, + ) + cfg.boost_body * _field_norm_tf( + tf_body, field_len=_blen(d), avg_field_len=avg_body_len, b_f=cfg.b_body ) + if tf_norm <= 0.0: + continue + + scores[d] += float(idf) * (tf_norm * k1p1 / (tf_norm + k1)) return scores diff --git a/src/backend/uv.lock b/src/backend/uv.lock index 689d70a..41d1033 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -56,6 +56,7 @@ dependencies = [ { name = "pydantic" }, { name = "requests" }, { name = "sentence-transformers" }, + { name = "stop-words" }, { name = "tqdm" }, { name = "uvicorn" }, ] @@ -81,6 +82,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.12.3" }, { name = "requests", specifier = ">=2.32.5" }, { name = "sentence-transformers", specifier = ">=5.2.3" }, + { name = "stop-words", specifier = ">=2025.11.4" }, { name = "tqdm", specifier = ">=4.67.1" }, { name = "uvicorn", specifier = ">=0.38.0" }, ] @@ -1259,6 +1261,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/e0/021c772d6a662f43b63044ab481dc6ac7592447605b5b35a957785363122/starlette-0.49.3-py3-none-any.whl", hash = "sha256:b579b99715fdc2980cf88c8ec96d3bf1ce16f5a8051a7c2b84ef9b1cdecaea2f", size = 74340, upload-time = "2025-11-01T15:12:24.387Z" }, ] +[[package]] +name = "stop-words" +version = "2025.11.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/cb/27ee3d3e0b7b1169269e83331c075b2dd3c4bcc1a005821174c32a273dc4/stop_words-2025.11.4.tar.gz", hash = "sha256:0459072b54b11e43a6fb4c5b05bda87d2accfc4f14c1697974f3739af0f7b43d", size = 68622, upload-time = "2025-11-03T21:07:42.359Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/f5/992d668d21590ed39c6a9d1c62220e9b4b086a165e15fcb7580764cc7ceb/stop_words-2025.11.4-py3-none-any.whl", hash = "sha256:b3fc0722e42b722a9350aad59a8ba5850085a5b45a4ba9de390b4f5c4b86df25", size = 59496, upload-time = "2025-11-03T21:07:41.14Z" }, +] + [[package]] name = "sympy" version = "1.14.0" From 4b6aafab53fe9de4f9ddc384908a5caf5c4d63b6 Mon Sep 17 00:00:00 2001 From: Jan Skowron Date: Fri, 27 Feb 2026 11:53:34 +0100 Subject: [PATCH 2/3] most funcs in cpp --- src/backend/bindings/utils.cpp | 287 +++++++++++++++++- .../search_engine/query/query_engine.py | 158 +--------- .../semantic_search/query_embeddings.py | 2 +- 3 files changed, 303 insertions(+), 144 deletions(-) diff --git a/src/backend/bindings/utils.cpp b/src/backend/bindings/utils.cpp index 300f562..3b51105 100644 --- a/src/backend/bindings/utils.cpp +++ b/src/backend/bindings/utils.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -211,6 +212,28 @@ struct PostingList { } }; +PostingList filter_posting_list(const PostingList* pl, + const std::unordered_set& allowed) { + PostingList result; + if (pl == nullptr) return result; + + for (uint32_t doc_id : pl->postings) { + if (allowed.count(doc_id)) { + result.postings.push_back(doc_id); + auto tf_it = pl->term_frequencies.find(doc_id); + if (tf_it != pl->term_frequencies.end()) { + result.term_frequencies[doc_id] = tf_it->second; + } + auto pos_it = pl->positions.find(doc_id); + if (pos_it != pl->positions.end()) { + result.positions[doc_id] = pos_it->second; + } + } + } + result.build_skip_pointers(); + return result; +} + PostingList read_posting_list(std::ifstream& in, uint64_t offset, uint32_t doc_freq) { PostingList pl; pl.doc_frequency = doc_freq; @@ -379,6 +402,19 @@ class InvertedIndex { return it->second; } + PostingList positional_phrase_search(const std::vector& terms, + const std::unordered_set& allowed); + + PostingList bool_search(py::object node, const std::vector& cand_list); + + std::unordered_map bm25_score_fielded( + const std::vector& query_terms, + const std::vector& candidate_doc_ids, + double k1, double boost_title, double boost_body, + double b_title, double b_body, + double idf_threshold, bool clamp_negative_idf, + int min_terms_after_threshold); + friend class DocStore; friend class IndexAccessor; }; @@ -875,6 +911,242 @@ PostingList positional_intersect(const PostingList& pl1, const PostingList& pl2, return result; } +PostingList InvertedIndex::positional_phrase_search( + const std::vector& terms, + const std::unordered_set& allowed) { + if (terms.empty()) { + return PostingList(); + } + + auto first_opt = index.get(terms[0]); + PostingList result = filter_posting_list( + first_opt ? &*first_opt : nullptr, allowed); + + if (result.postings.empty()) { + return PostingList(); + } + + for (size_t i = 1; i < terms.size(); ++i) { + auto next_opt = index.get(terms[i]); + PostingList next_pl = filter_posting_list( + next_opt ? &*next_opt : nullptr, allowed); + + if (next_pl.postings.empty()) { + // warm the cache for remaining terms (needed for snippet generation) + for (size_t j = i + 1; j < terms.size(); ++j) { + index.get(terms[j]); + } + return PostingList(); + } + + result = positional_intersect(result, next_pl, static_cast(i)); + + if (result.postings.empty()) { + for (size_t j = i + 1; j < terms.size(); ++j) { + index.get(terms[j]); + } + break; + } + } + + return result; +} + +PostingList bool_op_postings(const PostingList& left, const PostingList& right, + const std::string& op) { + std::unordered_set rset(right.postings.begin(), right.postings.end()); + + std::vector out; + + if (op == "AND") { + for (uint32_t d : left.postings) { + if (rset.count(d)) out.push_back(d); + } + } else if (op == "OR") { + std::unordered_set lset(left.postings.begin(), left.postings.end()); + out = left.postings; + for (uint32_t d : right.postings) { + if (!lset.count(d)) out.push_back(d); + } + } else if (op == "NOT") { + for (uint32_t d : left.postings) { + if (!rset.count(d)) out.push_back(d); + } + } + + std::sort(out.begin(), out.end()); + PostingList result; + result.postings = std::move(out); + return result; +} + +PostingList InvertedIndex::bool_search(py::object node, + const std::vector& cand_list) { + if (node.is_none()) return PostingList(); + + std::string value = node.attr("value").cast(); + + // leaf node (actual term, not an operator) + if (value != "AND" && value != "OR" && value != "NOT") { + auto pl_opt = index.get(value); + if (!pl_opt) return PostingList(); + + const auto& tf_map = pl_opt->term_frequencies; + std::vector out; + out.reserve(cand_list.size()); + for (uint32_t d : cand_list) { + if (tf_map.count(d)) out.push_back(d); + } + PostingList result; + result.postings = std::move(out); + return result; + } + + py::object py_left = node.attr("left"); + py::object py_right = node.attr("right"); + + if (value == "AND") { + bool left_is_not = !py_left.is_none() && + py_left.attr("value").cast() == "NOT"; + bool right_is_not = !py_right.is_none() && + py_right.attr("value").cast() == "NOT"; + + if (left_is_not) { + PostingList not_docs = bool_search( + py_left.is_none() ? py::none() : py_left.attr("right"), cand_list); + PostingList right_result = bool_search(py_right, cand_list); + return bool_op_postings(right_result, not_docs, "NOT"); + } + if (right_is_not) { + PostingList left_result = bool_search(py_left, cand_list); + PostingList not_docs = bool_search( + py_right.is_none() ? py::none() : py_right.attr("right"), cand_list); + return bool_op_postings(left_result, not_docs, "NOT"); + } + + PostingList left_result = bool_search(py_left, cand_list); + PostingList right_result = bool_search(py_right, cand_list); + return bool_op_postings(left_result, right_result, "AND"); + } + + // OR + PostingList left_result = bool_search(py_left, cand_list); + PostingList right_result = bool_search(py_right, cand_list); + return bool_op_postings(left_result, right_result, "OR"); +} + +inline double bm25_idf_cpp(uint32_t num_docs, uint32_t df, bool clamp_negative) { + if (num_docs == 0 || df == 0) return 0.0; + double val = std::log( + (static_cast(num_docs) - df + 0.5) / (df + 0.5)); + if (clamp_negative && val < 0.0) return 0.0; + return val; +} + +inline double field_norm_tf(int tf, int field_len, double avg_field_len, double b_f) { + if (tf <= 0) return 0.0; + if (avg_field_len <= 0.0) avg_field_len = 1.0; + if (field_len <= 0) field_len = 1; + double denom = (1.0 - b_f) + b_f * (static_cast(field_len) / avg_field_len); + if (denom <= 0.0) return static_cast(tf); + return static_cast(tf) / denom; +} + +std::unordered_map InvertedIndex::bm25_score_fielded( + const std::vector& query_terms, + const std::vector& candidate_doc_ids, + double k1, double boost_title, double boost_body, + double b_title, double b_body, + double idf_threshold, bool clamp_negative_idf, + int min_terms_after_threshold) { + + uint32_t num_docs = metadata.num_docs; + double avg_title_len = metadata.avg_title_length; + double avg_body_len = metadata.avg_body_length; + double k1p1 = k1 + 1.0; + + std::vector> term_idf_all; + term_idf_all.reserve(query_terms.size()); + for (const auto& t : query_terms) { + auto df_opt = get_docfreq(t); + if (!df_opt) continue; + double idf = bm25_idf_cpp(num_docs, *df_opt, clamp_negative_idf); + term_idf_all.push_back({t, idf}); + } + std::sort(term_idf_all.begin(), term_idf_all.end(), + [](const auto& a, const auto& b) { return a.second > b.second; }); + + // threshold filtering + std::vector> term_idf; + for (const auto& [t, idf] : term_idf_all) { + if (idf >= idf_threshold) term_idf.push_back({t, idf}); + } + int min_keep = std::max(1, min_terms_after_threshold); + if (static_cast(term_idf.size()) < min_keep) { + term_idf.clear(); + for (int i = 0; i < min_keep && i < static_cast(term_idf_all.size()); ++i) { + term_idf.push_back(term_idf_all[i]); + } + } + + std::unordered_set qterm_set(query_terms.begin(), query_terms.end()); + std::unordered_map> title_tf_cache; + + auto get_title_tf = [&](uint32_t doc_id, const std::string& term) -> int { + auto cache_it = title_tf_cache.find(doc_id); + if (cache_it == title_tf_cache.end()) { + auto title_opt = doc_store.get_title_only(doc_id); + std::string title = title_opt.value_or(""); + auto toks = normalize_search_query(title); + std::unordered_map counts; + for (const auto& tok : toks) { + if (qterm_set.count(tok)) counts[tok]++; + } + auto [inserted_it, _] = title_tf_cache.emplace(doc_id, std::move(counts)); + cache_it = inserted_it; + } + auto term_it = cache_it->second.find(term); + return term_it != cache_it->second.end() ? static_cast(term_it->second) : 0; + }; + + std::unordered_map scores; + scores.reserve(candidate_doc_ids.size()); + for (uint32_t d : candidate_doc_ids) { + scores[d] = 0.0; + } + + for (const auto& [t, idf] : term_idf) { + auto pl_opt = index.get(t); + if (!pl_opt) continue; + + const auto& tf_body_map = pl_opt->term_frequencies; + + for (uint32_t d : candidate_doc_ids) { + auto tf_it = tf_body_map.find(d); + if (tf_it == tf_body_map.end() || tf_it->second == 0) continue; + + int tf_body = static_cast(tf_it->second); + int tf_title = get_title_tf(d, t); + + double tf_norm = + boost_title * field_norm_tf( + tf_title, + static_cast(metadata.get_title_length(d)), + avg_title_len, b_title) + + boost_body * field_norm_tf( + tf_body, + static_cast(metadata.get_body_length(d)), + avg_body_len, b_body); + + if (tf_norm <= 0.0) continue; + + scores[d] += idf * (tf_norm * k1p1 / (tf_norm + k1)); + } + } + + return scores; +} + PostingList find_docs(const PostingList& pl1, const PostingList& pl2, const std::string& mode) { const auto& p1 = pl1.postings; const auto& p2 = pl2.postings; @@ -1056,5 +1328,18 @@ PYBIND11_MODULE(_core, m) { .def_readonly("metadata", &InvertedIndex::metadata) .def_readonly("doc_store", &InvertedIndex::doc_store) .def("clear_cache", &InvertedIndex::clear_cache) - .def("get_docfreq", &InvertedIndex::get_docfreq, py::arg("term")); + .def("get_docfreq", &InvertedIndex::get_docfreq, py::arg("term")) + .def("positional_phrase_search", &InvertedIndex::positional_phrase_search, + py::arg("terms"), py::arg("allowed"), + "Positional phrase search: intersect terms at consecutive positions, filtered by allowed doc IDs") + .def("bool_search", &InvertedIndex::bool_search, + py::arg("node"), py::arg("cand_list"), + "Recursive boolean search on a query tree Node, filtered by candidate doc IDs") + .def("bm25_score_fielded", &InvertedIndex::bm25_score_fielded, + py::arg("query_terms"), py::arg("candidate_doc_ids"), + py::arg("k1"), py::arg("boost_title"), py::arg("boost_body"), + py::arg("b_title"), py::arg("b_body"), + py::arg("idf_threshold"), py::arg("clamp_negative_idf"), + py::arg("min_terms_after_threshold"), + "Fielded BM25 scoring"); } diff --git a/src/backend/search_engine/query/query_engine.py b/src/backend/search_engine/query/query_engine.py index 176b284..6a6b36f 100644 --- a/src/backend/search_engine/query/query_engine.py +++ b/src/backend/search_engine/query/query_engine.py @@ -2,13 +2,11 @@ import heapq import time -from collections import Counter from dataclasses import dataclass from cpp_utils import ( # type: ignore [import-untyped] PostingList, normalize_search_query, - positional_intersect, ) from backend.logging_config import get_logger @@ -25,7 +23,6 @@ from backend.search_engine.scoring.bm25 import ( BM25Config, bm25_idf, - bm25_score_docs_fielded, stop_words, ) from backend.search_engine.semantic_search.query_embeddings import SemanticSearcher @@ -223,54 +220,7 @@ def _build_candidates_from_terms(self, terms: list[str]) -> list[int]: def _bool_search( self, node: Node | None, allowed: set[int], cand_list: list[int] ) -> PostingList: - start = time.perf_counter() - logger.debug(f"Evaluating node: {getattr(node, 'value', None)}") - - if node is None: - return self._empty_pl() - - if node.value not in (AND | OR | NOT): - pl = self.inverted_index.index.get(node.value) - if pl is None: - result = self._empty_pl() - else: - tf_map = pl.term_frequencies # dict-like: doc_id -> tf - out = [d for d in cand_list if d in tf_map] - result = PostingList(postings=out, term_frequencies={}, positions={}) - - elif node.value in AND: - left_is_not = node.left and node.left.value in NOT - right_is_not = node.right and node.right.value in NOT - - if left_is_not: - not_docs = self._bool_search( - node.left.right if node.left else None, allowed, cand_list - ) - right = self._bool_search(node.right, allowed, cand_list) - result = self._bool_op_postings(right, not_docs, "NOT") - - elif right_is_not: - left = self._bool_search(node.left, allowed, cand_list) - not_docs = self._bool_search( - node.right.right if node.right else None, allowed, cand_list - ) - result = self._bool_op_postings(left, not_docs, "NOT") - - else: - left = self._bool_search(node.left, allowed, cand_list) - right = self._bool_search(node.right, allowed, cand_list) - result = self._bool_op_postings(left, right, "AND") - - else: # OR - left = self._bool_search(node.left, allowed, cand_list) - right = self._bool_search(node.right, allowed, cand_list) - result = self._bool_op_postings(left, right, "OR") - - logger.debug( - f"Node={node.value!r}, Result docs={len(result.postings)}, " - f"Execution time: {time.perf_counter() - start:.6f} seconds" - ) - return result + return self.inverted_index.bool_search(node, cand_list) def _positional_phrase_search( self, terms: list[str], allowed: set[int] @@ -278,35 +228,7 @@ def _positional_phrase_search( start = time.perf_counter() logger.debug(f"Performing phrase search for: {terms}") - if not terms: - return self._empty_pl() - - first = self._filter_posting_list( - self.inverted_index.index.get(terms[0]), allowed - ) - if len(first.postings) == 0: - return self._empty_pl() - - result = first - for i, term in enumerate(terms[1:], start=1): - next_pl = self._filter_posting_list( - self.inverted_index.index.get(term), allowed - ) - if len(next_pl.postings) == 0: - return self._empty_pl() - - start_positional_intersect = time.perf_counter() - result = positional_intersect(result, next_pl, distance=i) - logger.debug( - f"Positional intersect for term '{term}' " - f"with distance={i} completed in " - f"{time.perf_counter() - start_positional_intersect:.6f}s" - ) - if len(result.postings) == 0: - # warm the cache for remaining terms (needed for snippet generation) - for remaining_term in terms[i + 1 :]: - self.inverted_index.index.get(remaining_term) - break + result = self.inverted_index.positional_phrase_search(terms, allowed) logger.debug( f"Result docs: {len(result.postings)}, " @@ -347,11 +269,7 @@ def search_results(self, limit: int = 10) -> SearchResults: qt = QueryTree() - t_norm = time.perf_counter() normalized_tokens = normalize_search_query(self._query) - logger.debug( - f"normalize_search_query time: {time.perf_counter() - t_norm:.6f}s" - ) logger.debug(f"Normalized search query: {normalized_tokens}") raw_query = self._query.strip() @@ -363,36 +281,19 @@ def search_results(self, limit: int = 10) -> SearchResults: ) base_terms = [t for t in normalized_tokens if t not in (AND | OR | NOT)] - - t_set_qt = time.perf_counter() self.inverted_index.doc_store.query_terms = list(set(base_terms)) - logger.debug( - f"set doc_store.query_terms (base) time: {time.perf_counter() - t_set_qt:.6f}s" - ) - t_flags = time.perf_counter() is_quoted_phrase = (raw_query.startswith('"') and raw_query.endswith('"')) or ( raw_query.startswith("'") and raw_query.endswith("'") ) - logger.debug(f"quoted phrase check time: {time.perf_counter() - t_flags:.6f}s") - t_has_ops = time.perf_counter() has_ops = qt._has_operators(normalized_tokens) - logger.debug(f"_has_operators time: {time.perf_counter() - t_has_ops:.6f}s") if has_ops: try: - t_parse_ops = time.perf_counter() qt.parse_query(normalized_tokens) - logger.debug( - f"parse_query (ops) time: {time.perf_counter() - t_parse_ops:.6f}s" - ) - t_set_qt2 = time.perf_counter() self.inverted_index.doc_store.query_terms = qt.unique_terms - logger.debug( - f"set doc_store.query_terms (ops) time: {time.perf_counter() - t_set_qt2:.6f}s" - ) except InvalidOperatorError as e: logger.error(f"Invalid query syntax: {e}") raise @@ -404,13 +305,13 @@ def search_results(self, limit: int = 10) -> SearchResults: ) query_terms = [term for term in _query_terms if term not in stop_words] + self.inverted_index.doc_store.query_terms = list(set(query_terms)) cand_terms = self._select_terms_for_candidates(query_terms) candidate_doc_ids = self._build_candidates_from_terms(cand_terms) cand_set = set(candidate_doc_ids) logger.debug(f"candidate_doc_ids count={len(candidate_doc_ids)}") - t_filter = time.perf_counter() restricted_result: PostingList if is_quoted_phrase: @@ -430,66 +331,39 @@ def search_results(self, limit: int = 10) -> SearchResults: logger.debug(f"Converted to AND query: {and_query}") qt2 = QueryTree() - t_parse_no_ops = time.perf_counter() qt2.parse_query(and_query) - logger.debug( - f"parse_query (no ops) time: {time.perf_counter() - t_parse_no_ops:.6f}s" - ) logger.debug(f"Query tree: {qt2.root}") t_bool = time.perf_counter() restricted_result = self._bool_search(qt2.root, cand_set, candidate_doc_ids) logger.debug(f"Bool search time: {time.perf_counter() - t_bool:.6f}s") - logger.debug(f"Filter time: {time.perf_counter() - t_filter:.6f}s") - has_boolean_results = ( restricted_result is not None and len(restricted_result.postings) > 0 ) - # scoring (Fielded BM25) + # scoring if has_boolean_results: logger.debug( f"Found {len(restricted_result.postings)} results " f"in {time.perf_counter() - start:.6f} seconds" ) t_score = time.perf_counter() - metadata = self.inverted_index.metadata - t_final_ids = time.perf_counter() - final_candidate_doc_ids = list(restricted_result.postings) - logger.debug( - f"final_candidate_doc_ids build time: {time.perf_counter() - t_final_ids:.6f}s" - ) - - # title TF cache: doc_id -> Counter(term->tf) for only query terms. - # IMPORTANT: uses get_title_only (no snippet IO). - qterm_set = set(query_terms) - title_tf_cache: dict[int, Counter[str]] = {} - - def get_title_tf(doc_id: int, term: str) -> int: - c = title_tf_cache.get(doc_id) - if c is None: - title_opt = self.inverted_index.doc_store.get_title_only( - int(doc_id) - ) - title = title_opt or "" - toks = normalize_search_query(title) # stem+tokenize in C++ - c = Counter(t for t in toks if t in qterm_set) - title_tf_cache[doc_id] = c - return int(c.get(term, 0)) + final_candidate_doc_ids = restricted_result.postings - scores = bm25_score_docs_fielded( + cfg = self.bm25_cfg + scores = self.inverted_index.bm25_score_fielded( query_terms=query_terms, - postings_by_term=self.inverted_index.index, # term -> PostingList candidate_doc_ids=final_candidate_doc_ids, - num_docs=int(metadata.num_docs), - avg_title_len=float(metadata.avg_title_length), - avg_body_len=float(metadata.avg_body_length), - get_title_len=metadata.get_title_length, - get_body_len=metadata.get_body_length, - get_title_tf=get_title_tf, - cfg=self.bm25_cfg, + k1=cfg.k1, + boost_title=cfg.boost_title, + boost_body=cfg.boost_body, + b_title=cfg.b_title, + b_body=cfg.b_body, + idf_threshold=cfg.idf_threshold, + clamp_negative_idf=cfg.clamp_negative_idf, + min_terms_after_threshold=cfg.min_terms_after_threshold, ) logger.debug( f"Fielded BM25 scoring time: {time.perf_counter() - t_score:.6f}s" @@ -570,5 +444,5 @@ def get_title_tf(doc_id: int, term: str) -> int: f"Total execution time: {end - start:.6f} seconds" ) # clear cache to free memory - self.inverted_index.clear_cache() + self.inverted_index.clear_cache() # TODO takes much time, zb in app.py auslagern nach response return SearchResults(search_results=search_results, correction=correction) diff --git a/src/backend/search_engine/semantic_search/query_embeddings.py b/src/backend/search_engine/semantic_search/query_embeddings.py index 5e35bfe..d37b8a0 100644 --- a/src/backend/search_engine/semantic_search/query_embeddings.py +++ b/src/backend/search_engine/semantic_search/query_embeddings.py @@ -22,7 +22,7 @@ def search(self, query: str, top_n): results = list(zip(ids[0], scores[0])) logger.debug( - f"Search found {len(results)} results in {time.perf_counter() - start:.4f}s" + f"Semantic search found {len(results)} results in {time.perf_counter() - start:.4f}s" ) return results From 82dc85774c5d8c2d2716b226550fae4f8a6a4d2d Mon Sep 17 00:00:00 2001 From: Jan Skowron Date: Fri, 27 Feb 2026 13:29:37 +0100 Subject: [PATCH 3/3] all models --- .gitignore | 4 +- src/backend/api/v1/app.py | 16 +- src/backend/pyproject.toml | 1 + .../ltr/build_data/build_dataset.py | 71 ++-- .../search_engine/ltr/build_data/features.py | 68 ++-- .../search_engine/ltr/build_data/io_utils.py | 16 +- .../search_engine/ltr/build_data/test.py | 12 - src/backend/search_engine/ltr/model.py | 62 +++ src/backend/search_engine/ltr/reranker.py | 108 ++++++ .../search_engine/ltr/train/__init__.py | 0 .../search_engine/ltr/train/train_ltr.py | 359 ++++++++++++++++++ .../search_engine/query/query_engine.py | 34 +- .../search_engine/query/query_engine_old.py | 305 --------------- src/backend/search_engine/scoring/bm25.py | 134 +------ src/backend/uv.lock | 166 ++++++++ 15 files changed, 827 insertions(+), 529 deletions(-) delete mode 100644 src/backend/search_engine/ltr/build_data/test.py create mode 100644 src/backend/search_engine/ltr/model.py create mode 100644 src/backend/search_engine/ltr/reranker.py create mode 100644 src/backend/search_engine/ltr/train/__init__.py create mode 100644 src/backend/search_engine/ltr/train/train_ltr.py delete mode 100755 src/backend/search_engine/query/query_engine_old.py diff --git a/.gitignore b/.gitignore index ea653ee..eadb4b7 100644 --- a/.gitignore +++ b/.gitignore @@ -242,6 +242,8 @@ dist-ssr !src/backend/search_engine/index_builder/data/data.md /src/backend/search_engine/index/bin/ src/backend/search_engine/models/IVFPQ.faiss +src/backend/search_engine/models/ltr_model.pt src/backend/search_engine/models/neuspell-scrnn-probwordnoise -src/backend/search_engine/ltr/build_data/data +src/backend/search_engine/ltr/data +src/backend/search_engine/ltr/train/output memory_log.txt diff --git a/src/backend/api/v1/app.py b/src/backend/api/v1/app.py index c4c88db..be139bc 100755 --- a/src/backend/api/v1/app.py +++ b/src/backend/api/v1/app.py @@ -14,7 +14,7 @@ ) from backend.search_engine.semantic_search.train_vector_index import train_or_load_ivfpq from backend.search_engine.spell_correction.spell_corrector import get_spell_corrector -from fastapi import FastAPI, HTTPException, Query, status +from fastapi import BackgroundTasks, FastAPI, HTTPException, Query, status from fastapi.middleware.cors import CORSMiddleware setup_logging(level=os.getenv("LOG_LEVEL", "INFO")) @@ -56,17 +56,27 @@ async def search( limit: Annotated[ int, Query(ge=1, le=500, description="Maximum number of results") ] = 10, + background_tasks: BackgroundTasks = BackgroundTasks(), ) -> SearchResults: - if app.state.inverted_index is None or app.state.spell_corrector is None: + if ( + app.state.inverted_index is None + or app.state.spell_corrector is None + or app.state.embedding_model is None + or app.state.vector_index is None + ): raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Search index or spell corrector not loaded", + detail="One or more models not loaded", ) try: qe = QueryEngine(q) results = qe.search_results(limit) + background_tasks.add_task( + qe.inverted_index.clear_cache + ) # clear snippet cache to reduce memory consumption + return results except InvalidOperatorError as e: raise HTTPException( diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index d050f28..feefb6a 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "sentence-transformers>=5.2.3", "psutil>=7.2.2", "stop-words>=2025.11.4", + "tensorboard>=2.20.0", ] [dependency-groups] diff --git a/src/backend/search_engine/ltr/build_data/build_dataset.py b/src/backend/search_engine/ltr/build_data/build_dataset.py index 1217d7d..94beba6 100644 --- a/src/backend/search_engine/ltr/build_data/build_dataset.py +++ b/src/backend/search_engine/ltr/build_data/build_dataset.py @@ -1,23 +1,26 @@ from __future__ import annotations import argparse +import gc import json import random from pathlib import Path +from cpp_utils import InvertedIndex # type: ignore from tqdm import tqdm -from .config import DatasetPaths, BuildConfig +from .config import BuildConfig, DatasetPaths +from .features import ( + build_postings_by_term, + compute_features_for_doc, + parse_query_terms, +) from .io_utils import ( iter_queries, - sample_qids_from_qrels, load_qrels_for_qids, load_top100_selection, + sample_qids_from_qrels, ) -from .features import parse_query_terms, build_postings_by_term, compute_features_for_doc - -# Your inverted index class from C++ bindings -from cpp_utils import InvertedIndex # type: ignore def split_qids(qids: list[int], *, seed: int) -> tuple[set[int], set[int], set[int]]: @@ -141,18 +144,34 @@ def write_jsonl(path: Path, rows, *, pretty: bool) -> None: def main() -> None: ap = argparse.ArgumentParser() - ap.add_argument("--data-dir", type=str, required=True, help="Folder containing doctrain-*.tsv") - ap.add_argument("--out-dir", type=str, required=True, help="Output folder for train/val/test jsonl") - ap.add_argument("--index-dir", type=str, required=True, help="Path to your built inverted index base dir") + ap.add_argument( + "--data-dir", type=str, required=True, help="Folder containing doctrain-*.tsv" + ) + ap.add_argument( + "--out-dir", + type=str, + required=True, + help="Output folder for train/val/test jsonl", + ) + ap.add_argument( + "--index-dir", + type=str, + required=True, + help="Path to your built inverted index base dir", + ) ap.add_argument("--seed", type=int, default=BuildConfig.seed) - ap.add_argument("--max-queries", type=int, default=100, help="Limit number of qids") + ap.add_argument("--max-queries", type=int, default=500, help="Limit number of qids") ap.add_argument("--hard-negatives", type=int, default=BuildConfig.hard_negatives) ap.add_argument("--soft-negatives", type=int, default=BuildConfig.soft_negatives) ap.add_argument("--soft-rank", type=int, default=BuildConfig.soft_rank) - ap.add_argument("--soft-fallback-from", type=int, default=BuildConfig.soft_fallback_from) - ap.add_argument("--soft-fallback-to", type=int, default=BuildConfig.soft_fallback_to) + ap.add_argument( + "--soft-fallback-from", type=int, default=BuildConfig.soft_fallback_from + ) + ap.add_argument( + "--soft-fallback-to", type=int, default=BuildConfig.soft_fallback_to + ) ap.add_argument("--pretty-json", action="store_true") @@ -173,11 +192,13 @@ def main() -> None: out_dir = Path(args.out_dir) paths = DatasetPaths.from_base(data_dir, out_dir) - # 1) sample qids from QRELS (guaranteed to have positives) - qid_list = sample_qids_from_qrels(paths.qrels_tsv, seed=cfg.seed, max_queries=cfg.max_queries) + # sample qids from QRELS (guaranteed to have positives) + qid_list = sample_qids_from_qrels( + paths.qrels_tsv, seed=cfg.seed, max_queries=cfg.max_queries + ) qids = set(qid_list) - # 1b) build qid->query map ONLY for those qids (streaming scan over queries.tsv) + # build qid->query map ONLY for those qids (streaming scan over queries.tsv) qid_to_query: dict[int, str] = {} missing = set(qids) @@ -193,13 +214,13 @@ def main() -> None: qids = set(qid_to_query.keys()) qid_list = list(qids) - # 2) split by qid (avoid leakage across query) + # split by qid (avoid leakage across query) train_qids, val_qids, test_qids = split_qids(qid_list, seed=cfg.seed) - # 3) load qrels only for selected qids + # load qrels only for selected qids qrels = load_qrels_for_qids(paths.qrels_tsv, qids) - # 4) load only needed top100 parts for selected qids + # load only needed top100 parts for selected qids top100_sel = load_top100_selection( paths.top100_tsv, qids, @@ -209,15 +230,13 @@ def main() -> None: soft_fallback_to=cfg.soft_fallback_to, ) - # 5) open index once + # open index once inverted_index = InvertedIndex(str(args.index_dir)) - # 6) streaming build: write train/val/test incrementally (RAM-efficient) out_dir.mkdir(parents=True, exist_ok=True) f_train = paths.train_jsonl.open("w", encoding="utf-8") f_val = paths.val_jsonl.open("w", encoding="utf-8") f_test = paths.test_jsonl.open("w", encoding="utf-8") - ##########DEBUG sk_no_qrels = 0 sk_no_top100 = 0 sk_no_query = 0 @@ -248,7 +267,6 @@ def main() -> None: soft_fallback_from=cfg.soft_fallback_from, soft_fallback_to=cfg.soft_fallback_to, ) - ##########DEBUG query = qid_to_query.get(qid) if query is None: sk_no_query += 1 @@ -265,7 +283,6 @@ def main() -> None: continue # ensure final count = 1 + hard + soft - # (If data quality issues cause fewer, we still write what we have.) row = build_one_example( inverted_index, qid=qid, @@ -273,7 +290,6 @@ def main() -> None: pos_doc=pos_doc, neg_docs=neg_docs, ) - #######DEBUG written += 1 s = json.dumps(row, ensure_ascii=False, separators=(",", ":")) if cfg.pretty_json: @@ -286,6 +302,11 @@ def main() -> None: else: f_test.write(s + "\n") + del row, s + + if written % 500 == 0: + gc.collect() + finally: f_train.close() f_val.close() @@ -304,4 +325,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/backend/search_engine/ltr/build_data/features.py b/src/backend/search_engine/ltr/build_data/features.py index df154df..4a47443 100644 --- a/src/backend/search_engine/ltr/build_data/features.py +++ b/src/backend/search_engine/ltr/build_data/features.py @@ -1,16 +1,9 @@ from __future__ import annotations from dataclasses import dataclass -from functools import lru_cache -from typing import Sequence, Mapping +from typing import Mapping, Sequence -# your C++ bindings module name might be: cpp_utils or similar. -# In your code snippet: "from cpp_utils import PostingList" -# and normalize_search_query is exported from _core. -# Adjust imports if needed. -from cpp_utils import normalize_search_query, PostingList # type: ignore - -from backend.search_engine.scoring.bm25 import bm25_score_docs_fielded, BM25Config +from cpp_utils import PostingList, normalize_search_query # type: ignore def parse_query_terms(query: str) -> list[str]: @@ -22,7 +15,9 @@ def parse_query_terms(query: str) -> list[str]: return [t for t in terms if t not in drop] -def build_postings_by_term(inverted_index, query_terms: Sequence[str]) -> dict[str, PostingList]: +def build_postings_by_term( + inverted_index, query_terms: Sequence[str] +) -> dict[str, PostingList]: postings: dict[str, PostingList] = {} for t in query_terms: pl = inverted_index.index.get(t) @@ -32,7 +27,9 @@ def build_postings_by_term(inverted_index, query_terms: Sequence[str]) -> dict[s return postings -def matched_terms_count(doc_id: int, postings_by_term: Mapping[str, PostingList]) -> int: +def matched_terms_count( + doc_id: int, postings_by_term: Mapping[str, PostingList] +) -> int: # how many query terms appear in doc (binary per term) c = 0 for t, pl in postings_by_term.items(): @@ -42,7 +39,9 @@ def matched_terms_count(doc_id: int, postings_by_term: Mapping[str, PostingList] return c -def phrase_match_indicator(doc_id: int, query_terms: Sequence[str], postings_by_term: Mapping[str, PostingList]) -> int: +def phrase_match_indicator( + doc_id: int, query_terms: Sequence[str], postings_by_term: Mapping[str, PostingList] +) -> int: """ Phrase match using positional postings: For terms t1 t2 ... tn, check existence of positions p, p+1, ..., p+n-1. @@ -65,25 +64,26 @@ def phrase_match_indicator(doc_id: int, query_terms: Sequence[str], postings_by_ # fast set-based progressive narrowing base = set(positions_lists[0]) # candidate start positions of first term for i in range(1, len(positions_lists)): - shifted = {p - i for p in positions_lists[i]} # positions where phrase could start + shifted = { + p - i for p in positions_lists[i] + } # positions where phrase could start base &= shifted if not base: return 0 return 1 -@lru_cache(maxsize=200_000) -def _cached_title_terms(inverted_index, doc_id: int) -> tuple[str, ...]: +def _get_title_terms(inverted_index, doc_id: int) -> tuple[str, ...]: title = inverted_index.doc_store.get_title_only(int(doc_id)) if not title: - return tuple() + return () terms = normalize_search_query(title) drop = {"AND", "OR", "NOT", "&", "|", "-", "(", ")"} return tuple(t for t in terms if t not in drop) def in_title_indicator(inverted_index, doc_id: int, query_terms: Sequence[str]) -> int: - title_terms = set(_cached_title_terms(inverted_index, int(doc_id))) + title_terms = set(_get_title_terms(inverted_index, int(doc_id))) for t in query_terms: if t in title_terms: return 1 @@ -92,7 +92,7 @@ def in_title_indicator(inverted_index, doc_id: int, query_terms: Sequence[str]) def title_tf(inverted_index, doc_id: int, term: str) -> int: # used by BM25 scorer; based on cached title terms - return int(_cached_title_terms(inverted_index, int(doc_id)).count(term)) + return int(_get_title_terms(inverted_index, int(doc_id)).count(term)) @dataclass(frozen=True) @@ -120,29 +120,17 @@ def compute_features_for_doc( query_terms: Sequence[str], postings_by_term: Mapping[str, PostingList], ) -> FeatureVector: - # BM25: compute body-only by setting title boost=0 - cfg = BM25Config( + scores = inverted_index.bm25_score_fielded( + query_terms=list(query_terms), + candidate_doc_ids=[int(doc_id)], + k1=1.2, boost_title=0.0, boost_body=1.0, - b_title=0.0, # irrelevant since boost_title=0 - b_body=BM25Config().b_body, - k1=BM25Config().k1, - idf_threshold=BM25Config().idf_threshold, - clamp_negative_idf=BM25Config().clamp_negative_idf, - min_terms_after_threshold=BM25Config().min_terms_after_threshold, - ) - - scores = bm25_score_docs_fielded( - list(query_terms), - postings_by_term=postings_by_term, - candidate_doc_ids=[int(doc_id)], - num_docs=int(inverted_index.metadata.num_docs), - avg_title_len=float(inverted_index.metadata.avg_title_length), - avg_body_len=float(inverted_index.metadata.avg_body_length), - get_title_len=lambda d: int(inverted_index.metadata.get_title_length(int(d))), - get_body_len=lambda d: int(inverted_index.metadata.get_body_length(int(d))), - get_title_tf=lambda d, t: int(title_tf(inverted_index, int(d), str(t))), - cfg=cfg, + b_title=0.0, + b_body=0.75, + idf_threshold=0.5, + clamp_negative_idf=True, + min_terms_after_threshold=1, ) bm25_body = float(scores.get(int(doc_id), 0.0)) @@ -159,4 +147,4 @@ def compute_features_for_doc( matched_frac=mf, phrase_match=pm, in_title=it, - ) \ No newline at end of file + ) diff --git a/src/backend/search_engine/ltr/build_data/io_utils.py b/src/backend/search_engine/ltr/build_data/io_utils.py index b594b9f..4675b2f 100644 --- a/src/backend/search_engine/ltr/build_data/io_utils.py +++ b/src/backend/search_engine/ltr/build_data/io_utils.py @@ -29,11 +29,12 @@ def iter_queries(path: Path) -> Iterable[Query]: continue -def sample_qids(queries_path: Path, *, seed: int, max_queries: int | None) -> list[Query]: +def sample_qids( + queries_path: Path, *, seed: int, max_queries: int | None +) -> list[Query]: rnd = random.Random(seed) if max_queries is None: - # wenn du wirklich ALLE willst, streamen wir trotzdem, aber ohne shuffle return list(iter_queries(queries_path)) k = int(max_queries) @@ -52,7 +53,10 @@ def sample_qids(queries_path: Path, *, seed: int, max_queries: int | None) -> li reservoir[j] = q return reservoir -def sample_qids_from_qrels(qrels_path: Path, *, seed: int, max_queries: int | None) -> list[int]: + +def sample_qids_from_qrels( + qrels_path: Path, *, seed: int, max_queries: int | None +) -> list[int]: """ Sample qids from doctrain-qrels.tsv (space-separated). Uses reservoir sampling if max_queries is set. @@ -185,8 +189,4 @@ def load_top100_selection( if soft_low <= rank <= soft_high: sel.soft_candidates[rank] = docid - # small early-stop optimization: if we already got all hard ranks - # and the whole soft bucket for this qid, we could stop per-qid, - # but doing that cleanly is messy; streaming is fast enough. - - return out \ No newline at end of file + return out diff --git a/src/backend/search_engine/ltr/build_data/test.py b/src/backend/search_engine/ltr/build_data/test.py deleted file mode 100644 index 5b73683..0000000 --- a/src/backend/search_engine/ltr/build_data/test.py +++ /dev/null @@ -1,12 +0,0 @@ -from backend.search_engine.index.index_loader import get_index - -inverted_index = get_index() -docstore = inverted_index.doc_store - -print(docstore.get_title_only(3175109)) - - -""" -uv run --project backend python -m backend.search_engine.ltr.build_data.build_dataset --data-dir /Users/janskowron/VSCode/search-engine/src/backend/search_engine/ltr/build_data/data/msmarco --out-dir /Users/janskowron/VSCode/search-engine/src/backend/search_engine/ltr/build_data/data/ltr_out --index-dir /Users/janskowron/VSCode/search-engine/src/backend/search_engine/index/bin - -""" diff --git a/src/backend/search_engine/ltr/model.py b/src/backend/search_engine/ltr/model.py new file mode 100644 index 0000000..19e222d --- /dev/null +++ b/src/backend/search_engine/ltr/model.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import torch +import torch.nn as nn + + +DEFAULT_FEATURE_ORDER = [ + "bm25_body", + "matched_terms", + "matched_frac", + "phrase_match", + "in_title", +] + + +class FeatureNormalizer(nn.Module): + """ + Stores mean/std as buffers -> saved with state_dict -> identical at serving time. + """ + + def __init__(self, num_features: int, eps: float = 1e-6, clip_z: float = 8.0): + super().__init__() + self.eps = eps + self.clip_z = clip_z + self.register_buffer("mean", torch.zeros(num_features)) + self.register_buffer("std", torch.ones(num_features)) + + def fit(self, x_all: torch.Tensor): + mean = x_all.mean(dim=0) + std = x_all.std(dim=0, unbiased=False).clamp_min(self.eps) + self.mean.copy_(mean) + self.std.copy_(std) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + z = (x - self.mean) / self.std + if self.clip_z is not None: + z = torch.clamp(z, -self.clip_z, self.clip_z) + return z + + +class TinyLTRModel(nn.Module): + """Lightweight per-doc scoring model (pointwise scoring, listwise loss).""" + + def __init__(self, num_features: int, hidden: int = 16, dropout: float = 0.1): + super().__init__() + self.norm = FeatureNormalizer(num_features=num_features) + self.mlp = nn.Sequential( + nn.Linear(num_features, hidden), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(hidden, 1), + ) + + def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + """ + x: (B, L, F) mask: (B, L) bool + returns scores: (B, L) + """ + x = self.norm(x) + s = self.mlp(x).squeeze(-1) # (B, L) + s = s.masked_fill(~mask, -1e9) + return s diff --git a/src/backend/search_engine/ltr/reranker.py b/src/backend/search_engine/ltr/reranker.py new file mode 100644 index 0000000..51000c8 --- /dev/null +++ b/src/backend/search_engine/ltr/reranker.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import time +from pathlib import Path + +import torch + +from backend.search_engine.ltr.model import TinyLTRModel, DEFAULT_FEATURE_ORDER +from backend.search_engine.ltr.build_data.features import ( + build_postings_by_term, + compute_features_for_doc, + parse_query_terms, +) +from backend.logging_config import get_logger + +logger = get_logger(__name__) + +_MODEL_PATH = Path(__file__).resolve().parents[1] / "models" / "ltr_model.pt" + + +class LTRReranker: + def __init__(self, model_path: Path = _MODEL_PATH) -> None: + logger.info(f"Loading LTR model from {model_path}") + checkpoint = torch.load(model_path, map_location="cpu", weights_only=False) + + self.feature_order: list[str] = checkpoint.get( + "feature_order", DEFAULT_FEATURE_ORDER + ) + hidden: int = checkpoint.get("hidden", 64) + dropout: float = checkpoint.get("dropout", 0.0) + + self.model = TinyLTRModel( + num_features=len(self.feature_order), + hidden=hidden, + dropout=dropout, + ) + self.model.load_state_dict(checkpoint["state_dict"]) + self.model.eval() + logger.info( + f"LTR model loaded (features={self.feature_order}, hidden={hidden})" + ) + + @torch.no_grad() + def rerank( + self, + ranked_candidates: list[tuple[int, float]], + raw_query: str, + inverted_index, + ) -> list[tuple[int, float]]: + """Re-rank BM25 candidates using the LTR model. + + Uses ``parse_query_terms`` from the training pipeline so that + feature computation is identical to how the training data was built. + """ + if not ranked_candidates: + return [] + + start = time.perf_counter() + + # same tokenisation as training (no stop-word removal, no boolean ops) + query_terms = parse_query_terms(raw_query) + if not query_terms: + return ranked_candidates + + postings_by_term = build_postings_by_term(inverted_index, query_terms) + + features_list: list[list[float]] = [] + doc_ids: list[int] = [] + for doc_id, _ in ranked_candidates: + doc_id = int(doc_id) + fv = compute_features_for_doc( + inverted_index, + doc_id=doc_id, + query_terms=query_terms, + postings_by_term=postings_by_term, + ) + feat_dict = fv.as_dict() + features_list.append( + [float(feat_dict.get(name, 0.0)) for name in self.feature_order] + ) + doc_ids.append(doc_id) + + x = torch.tensor([features_list], dtype=torch.float32) # (1, L, F) + mask = torch.ones(1, len(doc_ids), dtype=torch.bool) + + scores = self.model(x, mask).squeeze(0) # (L,) + + result = sorted( + zip(doc_ids, scores.tolist()), + key=lambda pair: pair[1], + reverse=True, + ) + + logger.debug( + f"LTR rerank time: {time.perf_counter() - start:.6f}s " + f"({len(doc_ids)} docs)" + ) + return result + + +_reranker: LTRReranker | None = None + + +def get_reranker() -> LTRReranker: + global _reranker + if _reranker is None: + _reranker = LTRReranker() + return _reranker diff --git a/src/backend/search_engine/ltr/train/__init__.py b/src/backend/search_engine/ltr/train/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/search_engine/ltr/train/train_ltr.py b/src/backend/search_engine/ltr/train/train_ltr.py new file mode 100644 index 0000000..d30432b --- /dev/null +++ b/src/backend/search_engine/ltr/train/train_ltr.py @@ -0,0 +1,359 @@ +# train_ltr.py +from __future__ import annotations + +import argparse +import gc +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from backend.search_engine.ltr.model import ( + DEFAULT_FEATURE_ORDER, + TinyLTRModel, +) +from torch.utils.data import DataLoader, Dataset +from torch.utils.tensorboard import SummaryWriter +from tqdm import tqdm + + +@dataclass +class Batch: + x: torch.Tensor # (B, L, F) + y: torch.Tensor # (B, L) + mask: torch.Tensor # (B, L) bool + qids: List[int] + + +class JsonlLTRDataset(Dataset): + """ + One line == one query impression: + {"qid":..., "query":..., "docs":[{"doc_id":..., "label":..., "features":{...}}, ...]} + """ + + def __init__( + self, path: Path, feature_order: List[str], max_docs: int | None = None + ): + self.path = Path(path) + self.feature_order = feature_order + self.max_docs = max_docs + + # Parse JSON and immediately convert to tensors — don't keep raw dicts + self.qids: List[int] = [] + self.xs: List[torch.Tensor] = [] + self.ys: List[torch.Tensor] = [] + + with self.path.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + row = json.loads(line) + docs = row["docs"] + if max_docs is not None: + docs = docs[:max_docs] + + feats = [] + labels = [] + for d in docs: + feat = d["features"] + feats.append([float(feat.get(name, 0.0)) for name in feature_order]) + labels.append(float(d.get("label", 0.0))) + + self.qids.append(int(row["qid"])) + self.xs.append(torch.tensor(feats, dtype=torch.float32)) + self.ys.append(torch.tensor(labels, dtype=torch.float32)) + # row goes out of scope here — GC can reclaim the dict + + def __len__(self) -> int: + return len(self.qids) + + def __getitem__(self, idx: int) -> Tuple[int, torch.Tensor, torch.Tensor]: + return self.qids[idx], self.xs[idx], self.ys[idx] + + +def collate_fn(batch_items: List[Tuple[int, torch.Tensor, torch.Tensor]]) -> Batch: + # pad to max L in batch + qids = [qid for (qid, _, _) in batch_items] + lengths = [x.shape[0] for (_, x, _) in batch_items] + max_l = max(lengths) + fdim = batch_items[0][1].shape[1] + + xs = torch.zeros((len(batch_items), max_l, fdim), dtype=torch.float32) + ys = torch.zeros((len(batch_items), max_l), dtype=torch.float32) + mask = torch.zeros((len(batch_items), max_l), dtype=torch.bool) + + for i, (_, x, y) in enumerate(batch_items): + l = x.shape[0] + xs[i, :l, :] = x + ys[i, :l] = y + mask[i, :l] = True + + return Batch(x=xs, y=ys, mask=mask, qids=qids) + + +@torch.no_grad() +def mrr_at_k( + scores: torch.Tensor, labels: torch.Tensor, mask: torch.Tensor, k: int = 10 +) -> float: + # scores/labels/mask: (B, L) + B, L = scores.shape + total = 0.0 + for b in range(B): + valid = mask[b] + s = scores[b][valid] + y = labels[b][valid] + if s.numel() == 0: + continue + + # sort desc + order = torch.argsort(s, descending=True) + y_sorted = y[order] + topk = y_sorted[: min(k, y_sorted.numel())] + + # first relevant + rr = 0.0 + for i in range(topk.numel()): + if topk[i].item() > 0: + rr = 1.0 / (i + 1) + break + total += rr + return total / B + + +@torch.no_grad() +def ndcg_at_k( + scores: torch.Tensor, labels: torch.Tensor, mask: torch.Tensor, k: int = 10 +) -> float: + # Binary labels are fine; formula matches slides. :contentReference[oaicite:6]{index=6} + B, L = scores.shape + total = 0.0 + for b in range(B): + valid = mask[b] + s = scores[b][valid] + y = labels[b][valid] + if s.numel() == 0: + continue + + order = torch.argsort(s, descending=True) + y_sorted = y[order] + topk = y_sorted[: min(k, y_sorted.numel())] + + # DCG + dcg = 0.0 + for i in range(topk.numel()): + rel = topk[i].item() + if rel > 0: + dcg += rel / math.log2( + 1.0 + (i + 1) + 0.0 + ) # log2(1+rank), rank is 1-based + + # IDCG + y_ideal = torch.sort(y, descending=True).values + ideal_topk = y_ideal[: min(k, y_ideal.numel())] + idcg = 0.0 + for i in range(ideal_topk.numel()): + rel = ideal_topk[i].item() + if rel > 0: + idcg += rel / math.log2(1.0 + (i + 1) + 0.0) + + total += (dcg / idcg) if idcg > 0 else 0.0 + + return total / B + + +def listwise_softmax_loss( + scores: torch.Tensor, labels: torch.Tensor, mask: torch.Tensor +) -> torch.Tensor: + """ + Implements: + L = - sum_i y_i * log softmax(s)_i + For multiple positives: normalize y to sum 1 over valid docs. + This corresponds to the listwise softmax loss shown in the slides. :contentReference[oaicite:8]{index=8} + """ + # scores already masked with -1e9 at pads, but keep mask for label normalization + y = labels.clone() + y = y.masked_fill(~mask, 0.0) + + # normalize labels per query (avoid all-zero) + denom = y.sum(dim=1, keepdim=True).clamp_min(1.0) + y = y / denom + + logp = F.log_softmax(scores, dim=1) + loss = -(y * logp).sum(dim=1).mean() + return loss + + +@torch.no_grad() +def evaluate( + model: TinyLTRModel, loader: DataLoader, device: torch.device +) -> Dict[str, float]: + model.eval() + mrrs, ndcgs, losses = [], [], [] + for batch in loader: + x = batch.x.to(device) + y = batch.y.to(device) + mask = batch.mask.to(device) + + scores = model(x, mask) + loss = listwise_softmax_loss(scores, y, mask) + + losses.append(loss.item()) + mrrs.append(mrr_at_k(scores, y, mask, k=10)) + ndcgs.append(ndcg_at_k(scores, y, mask, k=10)) + + return { + "loss": float(sum(losses) / max(1, len(losses))), + "mrr@10": float(sum(mrrs) / max(1, len(mrrs))), + "ndcg@10": float(sum(ndcgs) / max(1, len(ndcgs))), + } + + +def flatten_train_features(train_ds: JsonlLTRDataset) -> torch.Tensor: + # xs are already tensors stored in the dataset — concat directly + return torch.cat(train_ds.xs, dim=0) # (N, F) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--train", type=str, required=True) + ap.add_argument("--val", type=str, required=True) + ap.add_argument("--test", type=str, required=True) + + ap.add_argument( + "--max_docs", + type=int, + default=20, + help="truncate per query (see slides about truncation)", + ) + ap.add_argument("--hidden", type=int, default=256) + ap.add_argument("--dropout", type=float, default=0.0) + + ap.add_argument("--batch_size", type=int, default=1) + ap.add_argument("--lr", type=float, default=3e-4) + ap.add_argument("--epochs", type=int, default=100) + ap.add_argument("--grad_clip", type=float, default=1.0) + + _script_dir = Path(__file__).resolve().parent + ap.add_argument("--logdir", type=str, default=str(_script_dir / "output" / "runs")) + ap.add_argument( + "--out", type=str, default=str(_script_dir / "output" / "ltr_model.pt") + ) + + ap.add_argument("--seed", type=int, default=42) + ap.add_argument( + "--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu" + ) + args = ap.parse_args() + + torch.manual_seed(args.seed) + + feature_order = DEFAULT_FEATURE_ORDER + num_features = len(feature_order) + + train_ds = JsonlLTRDataset(Path(args.train), feature_order, max_docs=args.max_docs) + val_ds = JsonlLTRDataset(Path(args.val), feature_order, max_docs=args.max_docs) + test_ds = JsonlLTRDataset(Path(args.test), feature_order, max_docs=args.max_docs) + + train_loader = DataLoader( + train_ds, batch_size=args.batch_size, shuffle=True, collate_fn=collate_fn + ) + val_loader = DataLoader( + val_ds, batch_size=args.batch_size, shuffle=False, collate_fn=collate_fn + ) + test_loader = DataLoader( + test_ds, batch_size=args.batch_size, shuffle=False, collate_fn=collate_fn + ) + + device = torch.device(args.device) + + model = TinyLTRModel( + num_features=num_features, hidden=args.hidden, dropout=args.dropout + ).to(device) + + # Fit normalization on TRAIN set and store inside model buffers (saved with state_dict) + x_all = flatten_train_features(train_ds) # (N, F) + model.norm.fit(x_all) + del x_all + gc.collect() + + opt = torch.optim.Adam(model.parameters(), lr=args.lr) + + writer = SummaryWriter(log_dir=args.logdir) + + global_step = 0 + best_val = -1.0 + best_path = Path(args.out) + best_path.parent.mkdir(parents=True, exist_ok=True) + + for epoch in range(1, args.epochs + 1): + model.train() + pbar = tqdm(train_loader, desc=f"epoch {epoch}/{args.epochs}") + for batch in pbar: + x = batch.x.to(device) + y = batch.y.to(device) + mask = batch.mask.to(device) + + scores = model(x, mask) + loss = listwise_softmax_loss(scores, y, mask) + + opt.zero_grad(set_to_none=True) + loss.backward() + + if args.grad_clip is not None and args.grad_clip > 0: + nn.utils.clip_grad_norm_(model.parameters(), max_norm=args.grad_clip) + + opt.step() + + # track loss by iteration (as demanded) + writer.add_scalar("train/loss", loss.item(), global_step) + + if global_step % 200 == 0: + # quick train metrics on this batch (optional but useful) + with torch.no_grad(): + mrr = mrr_at_k(scores, y, mask, k=10) + ndcg = ndcg_at_k(scores, y, mask, k=10) + writer.add_scalar("train/mrr@10_batch", mrr, global_step) + writer.add_scalar("train/ndcg@10_batch", ndcg, global_step) + + pbar.set_postfix(loss=f"{loss.item():.4f}") + global_step += 1 + + if global_step % 500 == 0: + writer.flush() + + # full validation at epoch end (don’t look at test until the end) :contentReference[oaicite:10]{index=10} + val_metrics = evaluate(model, val_loader, device) + writer.add_scalar("val/loss", val_metrics["loss"], epoch) + writer.add_scalar("val/mrr@10", val_metrics["mrr@10"], epoch) + writer.add_scalar("val/ndcg@10", val_metrics["ndcg@10"], epoch) + + # choose best model by ndcg@10 (common) :contentReference[oaicite:11]{index=11} + if val_metrics["ndcg@10"] > best_val: + best_val = val_metrics["ndcg@10"] + payload = { + "state_dict": model.state_dict(), + "feature_order": feature_order, + "max_docs": args.max_docs, + "hidden": args.hidden, + "dropout": args.dropout, + } + torch.save(payload, best_path) + + print(f"[epoch {epoch}] val: {val_metrics}") + + best = torch.load(best_path, map_location=device) + model.load_state_dict(best["state_dict"]) + test_metrics = evaluate(model, test_loader, device) + print(f"[FINAL] test: {test_metrics}") + + writer.close() + print(f"Saved best model to: {best_path}") + + +if __name__ == "__main__": + main() diff --git a/src/backend/search_engine/query/query_engine.py b/src/backend/search_engine/query/query_engine.py index 6a6b36f..4e0cea9 100644 --- a/src/backend/search_engine/query/query_engine.py +++ b/src/backend/search_engine/query/query_engine.py @@ -2,7 +2,6 @@ import heapq import time -from dataclasses import dataclass from cpp_utils import ( # type: ignore [import-untyped] PostingList, @@ -22,6 +21,7 @@ ) from backend.search_engine.scoring.bm25 import ( BM25Config, + RetrievalConfig, bm25_idf, stop_words, ) @@ -29,19 +29,18 @@ from backend.search_engine.spell_correction.spell_correction import repl from backend.search_engine.spell_correction.spell_corrector import get_spell_corrector -logger = get_logger(__name__) +try: + from backend.search_engine.ltr import reranker as _ltr_module + _LTR_AVAILABLE = True +except ImportError: + _ltr_module = None # type: ignore[assignment] + _LTR_AVAILABLE = False -@dataclass(frozen=True) -class RetrievalConfig: - max_terms_for_candidates: int = 3 - max_candidates_total: int = 50_000 - max_candidates_per_term: int = 30_000 +logger = get_logger(__name__) - idf_threshold: float = 0.0 - min_terms_after_threshold: int = 1 - allow_fallback_full_retrieval: bool = True +# TODO: wild mix of query_terms in index for snippets class QueryEngine: @@ -277,7 +276,7 @@ def search_results(self, limit: int = 10) -> SearchResults: t_corr = time.perf_counter() correction = repl(self.corrector, raw_query) logger.debug( - f"spell correction total time: {time.perf_counter() - t_corr:.6f}s" + f"Spell correction total time: {time.perf_counter() - t_corr:.6f}s" ) base_terms = [t for t in normalized_tokens if t not in (AND | OR | NOT)] @@ -405,6 +404,17 @@ def search_results(self, limit: int = 10) -> SearchResults: lists=ranked_lists, top_n=limit, k=60 ) + # if _LTR_AVAILABLE and _ltr_module is not None: + # try: + # t_ltr = time.perf_counter() + # reranker = _ltr_module.get_reranker() + # final_top = reranker.rerank(final_top, raw_query, self.inverted_index) + # logger.debug(f"LTR rerank time: {time.perf_counter() - t_ltr:.6f}s") + # except Exception as e: + # logger.warning( + # f"LTR reranking failed, using BM25 + semantic ranking: {e}" + # ) + t_top = time.perf_counter() search_results: list[SearchResult] = [] @@ -443,6 +453,4 @@ def search_results(self, limit: int = 10) -> SearchResults: f"Returned {len(search_results)} results. " f"Total execution time: {end - start:.6f} seconds" ) - # clear cache to free memory - self.inverted_index.clear_cache() # TODO takes much time, zb in app.py auslagern nach response return SearchResults(search_results=search_results, correction=correction) diff --git a/src/backend/search_engine/query/query_engine_old.py b/src/backend/search_engine/query/query_engine_old.py deleted file mode 100755 index 8f39b96..0000000 --- a/src/backend/search_engine/query/query_engine_old.py +++ /dev/null @@ -1,305 +0,0 @@ -import heapq -import time - -from cpp_utils import ( # type: ignore [import-untyped] - PostingList, - find_docs, - normalize_search_query, - positional_intersect, -) - -from backend.logging_config import get_logger -from backend.search_engine.error_handling import InvalidOperatorError -from backend.search_engine.index.index_loader import get_index -from backend.search_engine.models.index import SearchResult, SearchResults -from backend.search_engine.query.query_preprocessing import ( - AND, - NOT, - OR, - Node, - QueryTree, -) -from backend.search_engine.scoring.bm25 import BM25Config, bm25_score_docs -from backend.search_engine.semantic_search.query_embeddings import SemanticSearcher -from backend.search_engine.spell_correction.spell_correction import repl -from backend.search_engine.spell_correction.spell_corrector import get_spell_corrector - -logger = get_logger(__name__) - - -class QueryEngine: - def __init__(self, q: str) -> None: - self._query = q - self.inverted_index = get_index() - self.corrector = get_spell_corrector() - self.semantic_searcher = SemanticSearcher() - - def _positional_phrase_search(self, terms: list[str]) -> PostingList: - start = time.perf_counter() - logger.debug(f"Performing phrase search for: {terms}") - - if not terms: - return PostingList(postings=[], term_frequencies={}, positions={}) - - result = self.inverted_index.index.get(terms[0]) - - if result is None: - return PostingList(postings=[], term_frequencies={}, positions={}) - - # for each subsequent term, check positions - for i, term in enumerate(terms[1:], start=1): - next_pl = self.inverted_index.index.get(term) - - if next_pl is None: - return PostingList(postings=[], term_frequencies={}, positions={}) - - start_positional_intersect = time.perf_counter() - result = positional_intersect(result, next_pl, distance=i) - logger.debug( - f"Positional intersect for term '{term}' " - f"with distance={i} completed in " - f"{time.perf_counter() - start_positional_intersect:.6f}s" - ) - if len(result.postings) == 0: - # warm the cache for remaining terms (needed for snippet generation) - for remaining_term in terms[i + 1 :]: - self.inverted_index.index.get(remaining_term) - break - - end = time.perf_counter() - logger.debug( - f"Result docs: {len(result.postings)}, " - f"Execution time: {end - start:.6f} seconds" - ) - - return result - - def _bool_search(self, node: Node | None) -> PostingList: - start = time.perf_counter() - logger.debug(f"Evaluating node: {getattr(node, 'value', None)}") - - if node is None: - return PostingList(postings=[], term_frequencies={}, positions={}) - - if node.value not in AND | OR | NOT: - pl = self.inverted_index.index.get(node.value) - result = pl or PostingList(postings=[], term_frequencies={}, positions={}) - - elif node.value in AND: - # check if one of the nodes has NOT child - left_is_not = node.left and node.left.value in NOT - right_is_not = node.right and node.right.value in NOT - - if left_is_not: - # NOT A AND B -> B minus A - not_docs = self._bool_search(node.left.right if node.left else None) - right = self._bool_search(node.right) - result = find_docs(right, not_docs, "NOT") - - elif right_is_not: - # A AND NOT B -> A minus B - left = self._bool_search(node.left) - not_docs = self._bool_search(node.right.right if node.right else None) - result = find_docs(left, not_docs, "NOT") - - else: - # regular AND without NOT - left = self._bool_search(node.left) - right = self._bool_search(node.right) - result = find_docs(left, right, "AND") - - else: # OR - left = self._bool_search(node.left) - right = self._bool_search(node.right) - result = find_docs(left, right, "OR") - - end = time.perf_counter() - logger.debug( - f"Node={node.value!r}, Result docs={len(result.postings)}, " - f"Execution time: {end - start:.6f} seconds" - ) - return result - - @staticmethod - def _to_boolean_normalized_query(tokens: list[str]) -> list[str]: - if not tokens: - return [] - - query_str = tokens[0] - - for term in tokens[1:]: - query_str = f"({query_str} AND {term})" - - return normalize_search_query(query_str) - - @staticmethod - def _reciprocal_rank_fusion( - lists: list[list[tuple[int, float]]], top_n: int, k: int - ) -> list[tuple[int, float]]: - """ - Fuse multiple ranked lists using Reciprocal Rank Fusion (RRF). - - Returns: - List of top_n (doc_id, combined_score) sorted by RRF score - """ - rrf_scores: dict[int, float] = {} - - for ranked_list in lists: - for rank, (doc_id, _) in enumerate(ranked_list): - rrf_scores[doc_id] = rrf_scores.get(doc_id, 0.0) + 1 / (k + rank + 1) - - return heapq.nlargest(top_n, rrf_scores.items(), key=lambda x: x[1]) - - def search_results(self, limit: int = 10) -> SearchResults: - start = time.perf_counter() - logger.debug("Starting query execution") - - qt = QueryTree() - normalized_tokens = normalize_search_query(self._query) - logger.debug(f"Normalized search query: {normalized_tokens}") - - raw_query = self._query.strip() - - correction = repl(self.corrector, raw_query) - - if not qt._has_operators(normalized_tokens): - self.inverted_index.doc_store.query_terms = list( - set(normalized_tokens) - ) # needed for snippets - if (raw_query.startswith('"') and raw_query.endswith('"')) or ( - raw_query.startswith("'") and raw_query.endswith("'") - ): - # positional phrase search - logger.debug("Executing positional phrase query search...") - normalized_tokens_no_quots = normalize_search_query(raw_query[1:-1]) - result = self._positional_phrase_search(normalized_tokens_no_quots) - else: - # any order -> create AND query - logger.debug("Executing phrase query search...") - and_query = QueryEngine._to_boolean_normalized_query(normalized_tokens) - logger.debug(f"Converted to AND query: {and_query}") - qt.parse_query(and_query) - logger.debug(f"Query tree: {qt.root}") - result = self._bool_search(qt.root) - else: - logger.debug("Executing bool query search...") - try: - qt.parse_query(normalized_tokens) - self.inverted_index.doc_store.query_terms = ( - qt.unique_terms - ) # needed for snippets - logger.debug(f"Query tree: {qt.root}") - result = self._bool_search(qt.root) - except InvalidOperatorError as e: - logger.error(f"Invalid query syntax: {e}") - raise - - has_boolean_results = result is not None and len(result.postings) > 0 - - if has_boolean_results: - logger.debug( - f"Found {len(result.postings)} results in {time.perf_counter() - start:.6f} seconds" - ) - - # candidates: bool/phrase search returns doc_ids in result.postings - candidate_doc_ids = list(result.postings) - metadata = self.inverted_index.metadata - - # score which query terms - # - bool queries: qt.unique_terms - # - AND-auto-query w/o operators: normalized_tokens - query_terms: list[str] = ( - list(qt.unique_terms) - if getattr(qt, "unique_terms", None) - else list(dict.fromkeys(normalized_tokens)) - ) - - t_bm25 = time.perf_counter() - - bm25_scores = bm25_score_docs( - query_terms=query_terms, - postings_by_term=self.inverted_index.index, # term -> PostingList - candidate_doc_ids=candidate_doc_ids, - num_docs=metadata.num_docs, - avgdl=metadata.avg_body_length, - get_doc_length=metadata.get_doc_length, - cfg=BM25Config( - k1=1.2, b=0.75, idf_threshold=0.0, clamp_negative_idf=True - ), - ) - - logger.debug(f"BM25 scoring time: {time.perf_counter() - t_bm25:.6f}s") - - t_sort_bm25 = time.perf_counter() - - # sort doc_ids acc to score - bm25_ranked_top = heapq.nlargest( - limit, - ( - (doc_id, bm25_scores.get(doc_id, 0.0)) - for doc_id in candidate_doc_ids - ), - key=lambda x: x[1], - ) - - logger.debug(f"Ranking sort time: {time.perf_counter() - t_sort_bm25:.6f}s") - else: - logger.debug( - "No boolean results found, falling back to semantic search only" - ) - bm25_ranked_top = [] - - t_semantic = time.perf_counter() - - semantic_scores = self.semantic_searcher.search(raw_query, limit * 2) - semantic_ranked_top = heapq.nlargest( - limit, - semantic_scores, - key=lambda x: x[1], - ) - logger.debug(f"Semantic ranking time: {time.perf_counter() - t_semantic:.6f}s") - - ranked_lists = [semantic_ranked_top] - if bm25_ranked_top: - ranked_lists.append(bm25_ranked_top) - - t_rrf = time.perf_counter() - final_top = QueryEngine._reciprocal_rank_fusion( - lists=ranked_lists, top_n=limit, k=60 - ) - logger.debug(f"RRF fusion time: {time.perf_counter() - t_rrf:.6f}s") - - search_results = [] - for doc_id, rrf_score in final_top: - doc_data = self.inverted_index.doc_store.get(doc_id) - if doc_data is None: - continue - - url = doc_data.url - if url is None: - continue - title = doc_data.title or "Untitled" - snippet = doc_data.snippet - - try: - search_result = SearchResult( - document_id=doc_id, - url=url, # type: ignore[arg-type] - title=title, - snippet=snippet, - rrf_score=rrf_score, - ) - search_results.append(search_result) - except Exception as e: - logger.error(f"Error creating SearchResult for doc_id {doc_id}: {e}") - continue - - end = time.perf_counter() - logger.debug( - f"Returned {len(search_results)} results. " - f"Total execution time: {end - start:.6f} seconds" - ) - # clear cache to free memory - self.inverted_index.clear_cache() - - return SearchResults(search_results=search_results, correction=correction) diff --git a/src/backend/search_engine/scoring/bm25.py b/src/backend/search_engine/scoring/bm25.py index 2230547..0352163 100644 --- a/src/backend/search_engine/scoring/bm25.py +++ b/src/backend/search_engine/scoring/bm25.py @@ -1,11 +1,7 @@ from __future__ import annotations import math -from collections.abc import Callable from dataclasses import dataclass -from typing import Iterable, Mapping, Sequence - -from cpp_utils import PostingList # type: ignore [import-untyped] stop_words = { "the", @@ -84,6 +80,18 @@ } +@dataclass(frozen=True) +class RetrievalConfig: + max_terms_for_candidates: int = 3 + max_candidates_total: int = 50_000 + max_candidates_per_term: int = 30_000 + + idf_threshold: float = 0.0 + min_terms_after_threshold: int = 1 + + allow_fallback_full_retrieval: bool = True + + @dataclass(frozen=True) class BM25Config: # global saturation parameter @@ -115,121 +123,3 @@ def bm25_idf(num_docs: int, df: int, *, clamp_negative: bool = True) -> float: if clamp_negative and val < 0.0: return 0.0 return val - - -def _field_norm_tf( - tf: int, *, field_len: int, avg_field_len: float, b_f: float -) -> float: - """ - Field-normalized TF: - tf / (1 - b_f + b_f * len_f(d) / avglen_f) - """ - if tf <= 0: - return 0.0 - if avg_field_len <= 0.0: - avg_field_len = 1.0 - if field_len <= 0: - field_len = 1 - - denom = (1.0 - b_f) + b_f * (float(field_len) / float(avg_field_len)) - if denom <= 0.0: - return float(tf) - return float(tf) / denom - - -def bm25_score_docs_fielded( - query_terms: Sequence[str], - *, - postings_by_term: Mapping[str, PostingList], - candidate_doc_ids: Iterable[int], - num_docs: int, - avg_title_len: float, - avg_body_len: float, - get_title_len: Callable[[int], int], - get_body_len: Callable[[int], int], - get_title_tf: Callable[[int, str], int], - cfg: BM25Config = BM25Config(), -) -> dict[int, float]: - cand_list = [int(d) for d in candidate_doc_ids] - - # --- idf compute once --- - term_idf_all: list[tuple[str, float]] = [] - for t in query_terms: - pl = postings_by_term.get(t) - if pl is None: - continue - df = getattr(pl, "doc_frequency", None) - if df is None: - df = len(pl.postings) - term_idf_all.append( - ( - t, - float( - bm25_idf(num_docs, int(df), clamp_negative=cfg.clamp_negative_idf) - ), - ) - ) - - term_idf_all.sort(key=lambda x: x[1], reverse=True) - - term_idf: dict[str, float] = { - t: idf for (t, idf) in term_idf_all if idf >= cfg.idf_threshold - } - min_keep = max(1, int(cfg.min_terms_after_threshold)) - if len(term_idf) < min_keep: - term_idf = {t: idf for (t, idf) in term_idf_all[:min_keep]} - - scores: dict[int, float] = {d: 0.0 for d in cand_list} - - # --- length caches --- - title_len_cache: dict[int, int] = {} - body_len_cache: dict[int, int] = {} - - def _tlen(d: int) -> int: - v = title_len_cache.get(d) - if v is None: - v = int(get_title_len(d)) - title_len_cache[d] = v - return v - - def _blen(d: int) -> int: - v = body_len_cache.get(d) - if v is None: - v = int(get_body_len(d)) - body_len_cache[d] = v - return v - - k1 = float(cfg.k1) - k1p1 = k1 + 1.0 - - # --- iterate candidates (FAST) --- - for t, idf in term_idf.items(): - pl = postings_by_term.get(t) - if pl is None: - continue - - tf_body_map = pl.term_frequencies # python mapping already - for d in cand_list: - tf_body = int(tf_body_map.get(d, 0)) - if tf_body <= 0: - # candidates are generated from body postings, so in practice tf_body>0 - # for the generating term; for OR-queries it may be 0 -> skip - continue - - # title tf only computed for docs we actually score - tf_title = int(get_title_tf(d, t)) - - tf_norm = cfg.boost_title * _field_norm_tf( - tf_title, - field_len=_tlen(d), - avg_field_len=avg_title_len, - b_f=cfg.b_title, - ) + cfg.boost_body * _field_norm_tf( - tf_body, field_len=_blen(d), avg_field_len=avg_body_len, b_f=cfg.b_body - ) - if tf_norm <= 0.0: - continue - - scores[d] += float(idf) * (tf_norm * k1p1 / (tf_norm + k1)) - - return scores diff --git a/src/backend/uv.lock b/src/backend/uv.lock index 41d1033..f11dc0d 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -2,6 +2,15 @@ version = 1 revision = 3 requires-python = ">=3.13" +[[package]] +name = "absl-py" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/64/c7/8de93764ad66968d19329a7e0c147a2bb3c7054c554d4a119111b8f9440f/absl_py-2.4.0.tar.gz", hash = "sha256:8c6af82722b35cf71e0f4d1d47dcaebfff286e27110a99fc359349b247dfb5d4", size = 116543, upload-time = "2026-01-28T10:17:05.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750, upload-time = "2026-01-28T10:17:04.19Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -57,6 +66,7 @@ dependencies = [ { name = "requests" }, { name = "sentence-transformers" }, { name = "stop-words" }, + { name = "tensorboard" }, { name = "tqdm" }, { name = "uvicorn" }, ] @@ -83,6 +93,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.5" }, { name = "sentence-transformers", specifier = ">=5.2.3" }, { name = "stop-words", specifier = ">=2025.11.4" }, + { name = "tensorboard", specifier = ">=2.20.0" }, { name = "tqdm", specifier = ">=4.67.1" }, { name = "uvicorn", specifier = ">=0.38.0" }, ] @@ -259,6 +270,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/c7/b64cae5dba3a1b138d7123ec36bb5ccd39d39939f18454407e5468f4763f/fsspec-2025.12.0-py3-none-any.whl", hash = "sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b", size = 201422, upload-time = "2025-12-03T15:23:41.434Z" }, ] +[[package]] +name = "grpcio" +version = "1.78.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/8a/3d098f35c143a89520e568e6539cc098fcd294495910e359889ce8741c84/grpcio-1.78.0.tar.gz", hash = "sha256:7382b95189546f375c174f53a5fa873cef91c4b8005faa05cc5b3beea9c4f1c5", size = 12852416, upload-time = "2026-02-06T09:57:18.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/a9/8f75894993895f361ed8636cd9237f4ab39ef87fd30db17467235ed1c045/grpcio-1.78.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:ce3a90455492bf8bfa38e56fbbe1dbd4f872a3d8eeaf7337dc3b1c8aa28c271b", size = 5920143, upload-time = "2026-02-06T09:55:52.035Z" }, + { url = "https://files.pythonhosted.org/packages/55/06/0b78408e938ac424100100fd081189451b472236e8a3a1f6500390dc4954/grpcio-1.78.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:2bf5e2e163b356978b23652c4818ce4759d40f4712ee9ec5a83c4be6f8c23a3a", size = 11803926, upload-time = "2026-02-06T09:55:55.494Z" }, + { url = "https://files.pythonhosted.org/packages/88/93/b59fe7832ff6ae3c78b813ea43dac60e295fa03606d14d89d2e0ec29f4f3/grpcio-1.78.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8f2ac84905d12918e4e55a16da17939eb63e433dc11b677267c35568aa63fc84", size = 6478628, upload-time = "2026-02-06T09:55:58.533Z" }, + { url = "https://files.pythonhosted.org/packages/ed/df/e67e3734527f9926b7d9c0dde6cd998d1d26850c3ed8eeec81297967ac67/grpcio-1.78.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b58f37edab4a3881bc6c9bca52670610e0c9ca14e2ea3cf9debf185b870457fb", size = 7173574, upload-time = "2026-02-06T09:56:01.786Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/cc03fffb07bfba982a9ec097b164e8835546980aec25ecfa5f9c1a47e022/grpcio-1.78.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:735e38e176a88ce41840c21bb49098ab66177c64c82426e24e0082500cc68af5", size = 6692639, upload-time = "2026-02-06T09:56:04.529Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9a/289c32e301b85bdb67d7ec68b752155e674ee3ba2173a1858f118e399ef3/grpcio-1.78.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2045397e63a7a0ee7957c25f7dbb36ddc110e0cfb418403d110c0a7a68a844e9", size = 7268838, upload-time = "2026-02-06T09:56:08.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/79/1be93f32add280461fa4773880196572563e9c8510861ac2da0ea0f892b6/grpcio-1.78.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:a9f136fbafe7ccf4ac7e8e0c28b31066e810be52d6e344ef954a3a70234e1702", size = 8251878, upload-time = "2026-02-06T09:56:10.914Z" }, + { url = "https://files.pythonhosted.org/packages/65/65/793f8e95296ab92e4164593674ae6291b204bb5f67f9d4a711489cd30ffa/grpcio-1.78.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:748b6138585379c737adc08aeffd21222abbda1a86a0dca2a39682feb9196c20", size = 7695412, upload-time = "2026-02-06T09:56:13.593Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9f/1e233fe697ecc82845942c2822ed06bb522e70d6771c28d5528e4c50f6a4/grpcio-1.78.0-cp313-cp313-win32.whl", hash = "sha256:271c73e6e5676afe4fc52907686670c7cea22ab2310b76a59b678403ed40d670", size = 4064899, upload-time = "2026-02-06T09:56:15.601Z" }, + { url = "https://files.pythonhosted.org/packages/4d/27/d86b89e36de8a951501fb06a0f38df19853210f341d0b28f83f4aa0ffa08/grpcio-1.78.0-cp313-cp313-win_amd64.whl", hash = "sha256:f2d4e43ee362adfc05994ed479334d5a451ab7bc3f3fee1b796b8ca66895acb4", size = 4797393, upload-time = "2026-02-06T09:56:17.882Z" }, + { url = "https://files.pythonhosted.org/packages/29/f2/b56e43e3c968bfe822fa6ce5bca10d5c723aa40875b48791ce1029bb78c7/grpcio-1.78.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:e87cbc002b6f440482b3519e36e1313eb5443e9e9e73d6a52d43bd2004fcfd8e", size = 5920591, upload-time = "2026-02-06T09:56:20.758Z" }, + { url = "https://files.pythonhosted.org/packages/5d/81/1f3b65bd30c334167bfa8b0d23300a44e2725ce39bba5b76a2460d85f745/grpcio-1.78.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:c41bc64626db62e72afec66b0c8a0da76491510015417c127bfc53b2fe6d7f7f", size = 11813685, upload-time = "2026-02-06T09:56:24.315Z" }, + { url = "https://files.pythonhosted.org/packages/0e/1c/bbe2f8216a5bd3036119c544d63c2e592bdf4a8ec6e4a1867592f4586b26/grpcio-1.78.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8dfffba826efcf366b1e3ccc37e67afe676f290e13a3b48d31a46739f80a8724", size = 6487803, upload-time = "2026-02-06T09:56:27.367Z" }, + { url = "https://files.pythonhosted.org/packages/16/5c/a6b2419723ea7ddce6308259a55e8e7593d88464ce8db9f4aa857aba96fa/grpcio-1.78.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:74be1268d1439eaaf552c698cdb11cd594f0c49295ae6bb72c34ee31abbe611b", size = 7173206, upload-time = "2026-02-06T09:56:29.876Z" }, + { url = "https://files.pythonhosted.org/packages/df/1e/b8801345629a415ea7e26c83d75eb5dbe91b07ffe5210cc517348a8d4218/grpcio-1.78.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be63c88b32e6c0f1429f1398ca5c09bc64b0d80950c8bb7807d7d7fb36fb84c7", size = 6693826, upload-time = "2026-02-06T09:56:32.305Z" }, + { url = "https://files.pythonhosted.org/packages/34/84/0de28eac0377742679a510784f049738a80424b17287739fc47d63c2439e/grpcio-1.78.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3c586ac70e855c721bda8f548d38c3ca66ac791dc49b66a8281a1f99db85e452", size = 7277897, upload-time = "2026-02-06T09:56:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/ca/9c/ad8685cfe20559a9edb66f735afdcb2b7d3de69b13666fdfc542e1916ebd/grpcio-1.78.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:35eb275bf1751d2ffbd8f57cdbc46058e857cf3971041521b78b7db94bdaf127", size = 8252404, upload-time = "2026-02-06T09:56:37.553Z" }, + { url = "https://files.pythonhosted.org/packages/3c/05/33a7a4985586f27e1de4803887c417ec7ced145ebd069bc38a9607059e2b/grpcio-1.78.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:207db540302c884b8848036b80db352a832b99dfdf41db1eb554c2c2c7800f65", size = 7696837, upload-time = "2026-02-06T09:56:40.173Z" }, + { url = "https://files.pythonhosted.org/packages/73/77/7382241caf88729b106e49e7d18e3116216c778e6a7e833826eb96de22f7/grpcio-1.78.0-cp314-cp314-win32.whl", hash = "sha256:57bab6deef2f4f1ca76cc04565df38dc5713ae6c17de690721bdf30cb1e0545c", size = 4142439, upload-time = "2026-02-06T09:56:43.258Z" }, + { url = "https://files.pythonhosted.org/packages/48/b2/b096ccce418882fbfda4f7496f9357aaa9a5af1896a9a7f60d9f2b275a06/grpcio-1.78.0-cp314-cp314-win_amd64.whl", hash = "sha256:dce09d6116df20a96acfdbf85e4866258c3758180e8c49845d6ba8248b6d0bbb", size = 4929852, upload-time = "2026-02-06T09:56:45.885Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -376,6 +418,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/62/d9ba6323b9202dd2fe166beab8a86d29465c41a0288cbe229fac60c1ab8d/jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55", size = 8701, upload-time = "2023-09-01T12:34:42.563Z" }, ] +[[package]] +name = "markdown" +version = "3.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -703,6 +754,64 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] +[[package]] +name = "pillow" +version = "12.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/42/5c74462b4fd957fcd7b13b04fb3205ff8349236ea74c7c375766d6c82288/pillow-12.1.1.tar.gz", hash = "sha256:9ad8fa5937ab05218e2b6a4cff30295ad35afd2f83ac592e68c0d871bb0fdbc4", size = 46980264, upload-time = "2026-02-11T04:23:07.146Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/11/6db24d4bd7685583caeae54b7009584e38da3c3d4488ed4cd25b439de486/pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d242e8ac078781f1de88bf823d70c1a9b3c7950a44cdf4b7c012e22ccbcd8e4e", size = 4062689, upload-time = "2026-02-11T04:21:06.804Z" }, + { url = "https://files.pythonhosted.org/packages/33/c0/ce6d3b1fe190f0021203e0d9b5b99e57843e345f15f9ef22fcd43842fd21/pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:02f84dfad02693676692746df05b89cf25597560db2857363a208e393429f5e9", size = 4138535, upload-time = "2026-02-11T04:21:08.452Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c6/d5eb6a4fb32a3f9c21a8c7613ec706534ea1cf9f4b3663e99f0d83f6fca8/pillow-12.1.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:e65498daf4b583091ccbb2556c7000abf0f3349fcd57ef7adc9a84a394ed29f6", size = 3601364, upload-time = "2026-02-11T04:21:10.194Z" }, + { url = "https://files.pythonhosted.org/packages/14/a1/16c4b823838ba4c9c52c0e6bbda903a3fe5a1bdbf1b8eb4fff7156f3e318/pillow-12.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c6db3b84c87d48d0088943bf33440e0c42370b99b1c2a7989216f7b42eede60", size = 5262561, upload-time = "2026-02-11T04:21:11.742Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ad/ad9dc98ff24f485008aa5cdedaf1a219876f6f6c42a4626c08bc4e80b120/pillow-12.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8b7e5304e34942bf62e15184219a7b5ad4ff7f3bb5cca4d984f37df1a0e1aee2", size = 4657460, upload-time = "2026-02-11T04:21:13.786Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f1a4ea9a895b5732152789326202a82464d5254759fbacae4deea3069334/pillow-12.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:18e5bddd742a44b7e6b1e773ab5db102bd7a94c32555ba656e76d319d19c3850", size = 6232698, upload-time = "2026-02-11T04:21:15.949Z" }, + { url = "https://files.pythonhosted.org/packages/95/f4/86f51b8745070daf21fd2e5b1fe0eb35d4db9ca26e6d58366562fb56a743/pillow-12.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc44ef1f3de4f45b50ccf9136999d71abb99dca7706bc75d222ed350b9fd2289", size = 8041706, upload-time = "2026-02-11T04:21:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/29/9b/d6ecd956bb1266dd1045e995cce9b8d77759e740953a1c9aad9502a0461e/pillow-12.1.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a8eb7ed8d4198bccbd07058416eeec51686b498e784eda166395a23eb99138e", size = 6346621, upload-time = "2026-02-11T04:21:19.547Z" }, + { url = "https://files.pythonhosted.org/packages/71/24/538bff45bde96535d7d998c6fed1a751c75ac7c53c37c90dc2601b243893/pillow-12.1.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47b94983da0c642de92ced1702c5b6c292a84bd3a8e1d1702ff923f183594717", size = 7038069, upload-time = "2026-02-11T04:21:21.378Z" }, + { url = "https://files.pythonhosted.org/packages/94/0e/58cb1a6bc48f746bc4cb3adb8cabff73e2742c92b3bf7a220b7cf69b9177/pillow-12.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:518a48c2aab7ce596d3bf79d0e275661b846e86e4d0e7dec34712c30fe07f02a", size = 6460040, upload-time = "2026-02-11T04:21:23.148Z" }, + { url = "https://files.pythonhosted.org/packages/6c/57/9045cb3ff11eeb6c1adce3b2d60d7d299d7b273a2e6c8381a524abfdc474/pillow-12.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a550ae29b95c6dc13cf69e2c9dc5747f814c54eeb2e32d683e5e93af56caa029", size = 7164523, upload-time = "2026-02-11T04:21:25.01Z" }, + { url = "https://files.pythonhosted.org/packages/73/f2/9be9cb99f2175f0d4dbadd6616ce1bf068ee54a28277ea1bf1fbf729c250/pillow-12.1.1-cp313-cp313-win32.whl", hash = "sha256:a003d7422449f6d1e3a34e3dd4110c22148336918ddbfc6a32581cd54b2e0b2b", size = 6332552, upload-time = "2026-02-11T04:21:27.238Z" }, + { url = "https://files.pythonhosted.org/packages/3f/eb/b0834ad8b583d7d9d42b80becff092082a1c3c156bb582590fcc973f1c7c/pillow-12.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:344cf1e3dab3be4b1fa08e449323d98a2a3f819ad20f4b22e77a0ede31f0faa1", size = 7040108, upload-time = "2026-02-11T04:21:29.462Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7d/fc09634e2aabdd0feabaff4a32f4a7d97789223e7c2042fd805ea4b4d2c2/pillow-12.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c0dd1636633e7e6a0afe7bf6a51a14992b7f8e60de5789018ebbdfae55b040a", size = 2453712, upload-time = "2026-02-11T04:21:31.072Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/b9d62794fc8a0dd14c1943df68347badbd5511103e0d04c035ffe5cf2255/pillow-12.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0330d233c1a0ead844fc097a7d16c0abff4c12e856c0b325f231820fee1f39da", size = 5264880, upload-time = "2026-02-11T04:21:32.865Z" }, + { url = "https://files.pythonhosted.org/packages/26/9d/e03d857d1347fa5ed9247e123fcd2a97b6220e15e9cb73ca0a8d91702c6e/pillow-12.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5dae5f21afb91322f2ff791895ddd8889e5e947ff59f71b46041c8ce6db790bc", size = 4660616, upload-time = "2026-02-11T04:21:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ec/8a6d22afd02570d30954e043f09c32772bfe143ba9285e2fdb11284952cd/pillow-12.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e0c664be47252947d870ac0d327fea7e63985a08794758aa8af5b6cb6ec0c9c", size = 6269008, upload-time = "2026-02-11T04:21:36.623Z" }, + { url = "https://files.pythonhosted.org/packages/3d/1d/6d875422c9f28a4a361f495a5f68d9de4a66941dc2c619103ca335fa6446/pillow-12.1.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:691ab2ac363b8217f7d31b3497108fb1f50faab2f75dfb03284ec2f217e87bf8", size = 8073226, upload-time = "2026-02-11T04:21:38.585Z" }, + { url = "https://files.pythonhosted.org/packages/a1/cd/134b0b6ee5eda6dc09e25e24b40fdafe11a520bc725c1d0bbaa5e00bf95b/pillow-12.1.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9e8064fb1cc019296958595f6db671fba95209e3ceb0c4734c9baf97de04b20", size = 6380136, upload-time = "2026-02-11T04:21:40.562Z" }, + { url = "https://files.pythonhosted.org/packages/7a/a9/7628f013f18f001c1b98d8fffe3452f306a70dc6aba7d931019e0492f45e/pillow-12.1.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:472a8d7ded663e6162dafdf20015c486a7009483ca671cece7a9279b512fcb13", size = 7067129, upload-time = "2026-02-11T04:21:42.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f8/66ab30a2193b277785601e82ee2d49f68ea575d9637e5e234faaa98efa4c/pillow-12.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:89b54027a766529136a06cfebeecb3a04900397a3590fd252160b888479517bf", size = 6491807, upload-time = "2026-02-11T04:21:44.22Z" }, + { url = "https://files.pythonhosted.org/packages/da/0b/a877a6627dc8318fdb84e357c5e1a758c0941ab1ddffdafd231983788579/pillow-12.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:86172b0831b82ce4f7877f280055892b31179e1576aa00d0df3bb1bbf8c3e524", size = 7190954, upload-time = "2026-02-11T04:21:46.114Z" }, + { url = "https://files.pythonhosted.org/packages/83/43/6f732ff85743cf746b1361b91665d9f5155e1483817f693f8d57ea93147f/pillow-12.1.1-cp313-cp313t-win32.whl", hash = "sha256:44ce27545b6efcf0fdbdceb31c9a5bdea9333e664cda58a7e674bb74608b3986", size = 6336441, upload-time = "2026-02-11T04:21:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/3b/44/e865ef3986611bb75bfabdf94a590016ea327833f434558801122979cd0e/pillow-12.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a285e3eb7a5a45a2ff504e31f4a8d1b12ef62e84e5411c6804a42197c1cf586c", size = 7045383, upload-time = "2026-02-11T04:21:50.015Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c6/f4fb24268d0c6908b9f04143697ea18b0379490cb74ba9e8d41b898bd005/pillow-12.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:cc7d296b5ea4d29e6570dabeaed58d31c3fea35a633a69679fb03d7664f43fb3", size = 2456104, upload-time = "2026-02-11T04:21:51.633Z" }, + { url = "https://files.pythonhosted.org/packages/03/d0/bebb3ffbf31c5a8e97241476c4cf8b9828954693ce6744b4a2326af3e16b/pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:417423db963cb4be8bac3fc1204fe61610f6abeed1580a7a2cbb2fbda20f12af", size = 4062652, upload-time = "2026-02-11T04:21:53.19Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c0/0e16fb0addda4851445c28f8350d8c512f09de27bbb0d6d0bbf8b6709605/pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:b957b71c6b2387610f556a7eb0828afbe40b4a98036fc0d2acfa5a44a0c2036f", size = 4138823, upload-time = "2026-02-11T04:22:03.088Z" }, + { url = "https://files.pythonhosted.org/packages/6b/fb/6170ec655d6f6bb6630a013dd7cf7bc218423d7b5fa9071bf63dc32175ae/pillow-12.1.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:097690ba1f2efdeb165a20469d59d8bb03c55fb6621eb2041a060ae8ea3e9642", size = 3601143, upload-time = "2026-02-11T04:22:04.909Z" }, + { url = "https://files.pythonhosted.org/packages/59/04/dc5c3f297510ba9a6837cbb318b87dd2b8f73eb41a43cc63767f65cb599c/pillow-12.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2815a87ab27848db0321fb78c7f0b2c8649dee134b7f2b80c6a45c6831d75ccd", size = 5266254, upload-time = "2026-02-11T04:22:07.656Z" }, + { url = "https://files.pythonhosted.org/packages/05/30/5db1236b0d6313f03ebf97f5e17cda9ca060f524b2fcc875149a8360b21c/pillow-12.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f7ed2c6543bad5a7d5530eb9e78c53132f93dfa44a28492db88b41cdab885202", size = 4657499, upload-time = "2026-02-11T04:22:09.613Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/008d2ca0eb612e81968e8be0bbae5051efba24d52debf930126d7eaacbba/pillow-12.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:652a2c9ccfb556235b2b501a3a7cf3742148cd22e04b5625c5fe057ea3e3191f", size = 6232137, upload-time = "2026-02-11T04:22:11.434Z" }, + { url = "https://files.pythonhosted.org/packages/70/f1/f14d5b8eeb4b2cd62b9f9f847eb6605f103df89ef619ac68f92f748614ea/pillow-12.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d6e4571eedf43af33d0fc233a382a76e849badbccdf1ac438841308652a08e1f", size = 8042721, upload-time = "2026-02-11T04:22:13.321Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d6/17824509146e4babbdabf04d8171491fa9d776f7061ff6e727522df9bd03/pillow-12.1.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b574c51cf7d5d62e9be37ba446224b59a2da26dc4c1bb2ecbe936a4fb1a7cb7f", size = 6347798, upload-time = "2026-02-11T04:22:15.449Z" }, + { url = "https://files.pythonhosted.org/packages/d1/ee/c85a38a9ab92037a75615aba572c85ea51e605265036e00c5b67dfafbfe2/pillow-12.1.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a37691702ed687799de29a518d63d4682d9016932db66d4e90c345831b02fb4e", size = 7039315, upload-time = "2026-02-11T04:22:17.24Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f3/bc8ccc6e08a148290d7523bde4d9a0d6c981db34631390dc6e6ec34cacf6/pillow-12.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f95c00d5d6700b2b890479664a06e754974848afaae5e21beb4d83c106923fd0", size = 6462360, upload-time = "2026-02-11T04:22:19.111Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ab/69a42656adb1d0665ab051eec58a41f169ad295cf81ad45406963105408f/pillow-12.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:559b38da23606e68681337ad74622c4dbba02254fc9cb4488a305dd5975c7eeb", size = 7165438, upload-time = "2026-02-11T04:22:21.041Z" }, + { url = "https://files.pythonhosted.org/packages/02/46/81f7aa8941873f0f01d4b55cc543b0a3d03ec2ee30d617a0448bf6bd6dec/pillow-12.1.1-cp314-cp314-win32.whl", hash = "sha256:03edcc34d688572014ff223c125a3f77fb08091e4607e7745002fc214070b35f", size = 6431503, upload-time = "2026-02-11T04:22:22.833Z" }, + { url = "https://files.pythonhosted.org/packages/40/72/4c245f7d1044b67affc7f134a09ea619d4895333d35322b775b928180044/pillow-12.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:50480dcd74fa63b8e78235957d302d98d98d82ccbfac4c7e12108ba9ecbdba15", size = 7176748, upload-time = "2026-02-11T04:22:24.64Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ad/8a87bdbe038c5c698736e3348af5c2194ffb872ea52f11894c95f9305435/pillow-12.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:5cb1785d97b0c3d1d1a16bc1d710c4a0049daefc4935f3a8f31f827f4d3d2e7f", size = 2544314, upload-time = "2026-02-11T04:22:26.685Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9d/efd18493f9de13b87ede7c47e69184b9e859e4427225ea962e32e56a49bc/pillow-12.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1f90cff8aa76835cba5769f0b3121a22bd4eb9e6884cfe338216e557a9a548b8", size = 5268612, upload-time = "2026-02-11T04:22:29.884Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f1/4f42eb2b388eb2ffc660dcb7f7b556c1015c53ebd5f7f754965ef997585b/pillow-12.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1f1be78ce9466a7ee64bfda57bdba0f7cc499d9794d518b854816c41bf0aa4e9", size = 4660567, upload-time = "2026-02-11T04:22:31.799Z" }, + { url = "https://files.pythonhosted.org/packages/01/54/df6ef130fa43e4b82e32624a7b821a2be1c5653a5fdad8469687a7db4e00/pillow-12.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:42fc1f4677106188ad9a55562bbade416f8b55456f522430fadab3cef7cd4e60", size = 6269951, upload-time = "2026-02-11T04:22:33.921Z" }, + { url = "https://files.pythonhosted.org/packages/a9/48/618752d06cc44bb4aae8ce0cd4e6426871929ed7b46215638088270d9b34/pillow-12.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98edb152429ab62a1818039744d8fbb3ccab98a7c29fc3d5fcef158f3f1f68b7", size = 8074769, upload-time = "2026-02-11T04:22:35.877Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bd/f1d71eb39a72fa088d938655afba3e00b38018d052752f435838961127d8/pillow-12.1.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d470ab1178551dd17fdba0fef463359c41aaa613cdcd7ff8373f54be629f9f8f", size = 6381358, upload-time = "2026-02-11T04:22:37.698Z" }, + { url = "https://files.pythonhosted.org/packages/64/ef/c784e20b96674ed36a5af839305f55616f8b4f8aa8eeccf8531a6e312243/pillow-12.1.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6408a7b064595afcab0a49393a413732a35788f2a5092fdc6266952ed67de586", size = 7068558, upload-time = "2026-02-11T04:22:39.597Z" }, + { url = "https://files.pythonhosted.org/packages/73/cb/8059688b74422ae61278202c4e1ad992e8a2e7375227be0a21c6b87ca8d5/pillow-12.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5d8c41325b382c07799a3682c1c258469ea2ff97103c53717b7893862d0c98ce", size = 6493028, upload-time = "2026-02-11T04:22:42.73Z" }, + { url = "https://files.pythonhosted.org/packages/c6/da/e3c008ed7d2dd1f905b15949325934510b9d1931e5df999bb15972756818/pillow-12.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7697918b5be27424e9ce568193efd13d925c4481dd364e43f5dff72d33e10f8", size = 7191940, upload-time = "2026-02-11T04:22:44.543Z" }, + { url = "https://files.pythonhosted.org/packages/01/4a/9202e8d11714c1fc5951f2e1ef362f2d7fbc595e1f6717971d5dd750e969/pillow-12.1.1-cp314-cp314t-win32.whl", hash = "sha256:d2912fd8114fc5545aa3a4b5576512f64c55a03f3ebcca4c10194d593d43ea36", size = 6438736, upload-time = "2026-02-11T04:22:46.347Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ca/cbce2327eb9885476b3957b2e82eb12c866a8b16ad77392864ad601022ce/pillow-12.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4ceb838d4bd9dab43e06c363cab2eebf63846d6a4aeaea283bbdfd8f1a8ed58b", size = 7182894, upload-time = "2026-02-11T04:22:48.114Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d2/de599c95ba0a973b94410477f8bf0b6f0b5e67360eb89bcb1ad365258beb/pillow-12.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7b03048319bfc6170e93bd60728a1af51d3dd7704935feb228c4d4faab35d334", size = 2546446, upload-time = "2026-02-11T04:22:50.342Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -740,6 +849,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/18/72c216f4ab0c82b907009668f79183ae029116ff0dd245d56ef58aac48e7/polars_runtime_32-1.38.1-cp310-abi3-win_arm64.whl", hash = "sha256:6d07d0cc832bfe4fb54b6e04218c2c27afcfa6b9498f9f6bbf262a00d58cc7c4", size = 41639413, upload-time = "2026-02-06T18:12:22.044Z" }, ] +[[package]] +name = "protobuf" +version = "7.34.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/00/04a2ab36b70a52d0356852979e08b44edde0435f2115dc66e25f2100f3ab/protobuf-7.34.0.tar.gz", hash = "sha256:3871a3df67c710aaf7bb8d214cc997342e63ceebd940c8c7fc65c9b3d697591a", size = 454726, upload-time = "2026-02-27T00:30:25.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/c4/6322ab5c8f279c4c358bc14eb8aefc0550b97222a39f04eb3c1af7a830fa/protobuf-7.34.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e329966799f2c271d5e05e236459fe1cbfdb8755aaa3b0914fa60947ddea408", size = 429248, upload-time = "2026-02-27T00:30:14.924Z" }, + { url = "https://files.pythonhosted.org/packages/45/99/b029bbbc61e8937545da5b79aa405ab2d9cf307a728f8c9459ad60d7a481/protobuf-7.34.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:9d7a5005fb96f3c1e64f397f91500b0eb371b28da81296ae73a6b08a5b76cdd6", size = 325753, upload-time = "2026-02-27T00:30:17.247Z" }, + { url = "https://files.pythonhosted.org/packages/cc/79/09f02671eb75b251c5550a1c48e7b3d4b0623efd7c95a15a50f6f9fc1e2e/protobuf-7.34.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:4a72a8ec94e7a9f7ef7fe818ed26d073305f347f8b3b5ba31e22f81fd85fca02", size = 340200, upload-time = "2026-02-27T00:30:18.672Z" }, + { url = "https://files.pythonhosted.org/packages/b5/57/89727baef7578897af5ed166735ceb315819f1c184da8c3441271dbcfde7/protobuf-7.34.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:964cf977e07f479c0697964e83deda72bcbc75c3badab506fb061b352d991b01", size = 324268, upload-time = "2026-02-27T00:30:20.088Z" }, + { url = "https://files.pythonhosted.org/packages/1f/3e/38ff2ddee5cc946f575c9d8cc822e34bde205cf61acf8099ad88ef19d7d2/protobuf-7.34.0-cp310-abi3-win32.whl", hash = "sha256:f791ec509707a1d91bd02e07df157e75e4fb9fbdad12a81b7396201ec244e2e3", size = 426628, upload-time = "2026-02-27T00:30:21.555Z" }, + { url = "https://files.pythonhosted.org/packages/cb/71/7c32eaf34a61a1bae1b62a2ac4ffe09b8d1bb0cf93ad505f42040023db89/protobuf-7.34.0-cp310-abi3-win_amd64.whl", hash = "sha256:9f9079f1dde4e32342ecbd1c118d76367090d4aaa19da78230c38101c5b3dd40", size = 437901, upload-time = "2026-02-27T00:30:22.836Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e7/14dc9366696dcb53a413449881743426ed289d687bcf3d5aee4726c32ebb/protobuf-7.34.0-py3-none-any.whl", hash = "sha256:e3b914dd77fa33fa06ab2baa97937746ab25695f389869afdf03e81f34e45dc7", size = 170716, upload-time = "2026-02-27T00:30:23.994Z" }, +] + [[package]] name = "psutil" version = "7.2.2" @@ -1282,6 +1406,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] +[[package]] +name = "tensorboard" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "grpcio" }, + { name = "markdown" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "setuptools" }, + { name = "tensorboard-data-server" }, + { name = "werkzeug" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" }, +] + +[[package]] +name = "tensorboard-data-server" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/dabeaf902892922777492e1d253bb7e1264cadce3cea932f7ff599e53fea/tensorboard_data_server-0.7.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:9fe5d24221b29625dbc7328b0436ca7fc1c23de4acf4d272f1180856e32f9f60", size = 4823598, upload-time = "2023-10-23T21:23:33.714Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" }, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -1474,3 +1628,15 @@ sdist = { url = "https://files.pythonhosted.org/packages/cb/ce/f06b84e2697fef468 wheels = [ { url = "https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02", size = 68109, upload-time = "2025-10-18T13:46:42.958Z" }, ] + +[[package]] +name = "werkzeug" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/f1/ee81806690a87dab5f5653c1f146c92bc066d7f4cebc603ef88eb9e13957/werkzeug-3.1.6.tar.gz", hash = "sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25", size = 864736, upload-time = "2026-02-19T15:17:18.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl", hash = "sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131", size = 225166, upload-time = "2026-02-19T15:17:17.475Z" }, +]