diff --git a/MPCAutofill/cardpicker/search/search_functions.py b/MPCAutofill/cardpicker/search/search_functions.py index a2baf12a3..a6726a7dd 100644 --- a/MPCAutofill/cardpicker/search/search_functions.py +++ b/MPCAutofill/cardpicker/search/search_functions.py @@ -5,6 +5,7 @@ import pycountry from elasticsearch import Elasticsearch from elasticsearch.exceptions import ConnectionError as ElasticConnectionError +from elasticsearch.exceptions import TransportError as ElasticTransportError from elasticsearch_dsl.query import Bool, Match, Range, Terms from django.conf import settings @@ -88,18 +89,39 @@ def get_scaled_maximum_size(search_settings: SearchSettings) -> int: return search_settings.filterSettings.maximumSize * 1_000_000 +# fallback levels for the fuzzy retry-on-miss behaviour. level 0 is the standard match; levels 1 and 2 are +# progressively more forgiving and are only queried when every stricter level matched nothing. +FUZZY_FALLBACK_LEVELS = [1, 2] + + +def get_match(search_settings: SearchSettings, query_parsed: str, fallback_level: int = 0) -> Match: + if fallback_level == 1: + # forgive typos (up to elasticsearch's automatic edit distance), but still require every word to match + return Match(searchq_fuzzy={"query": query_parsed, "operator": "AND", "fuzziness": "AUTO"}) + if fallback_level == 2: + # forgive typos and extra words. "2<75%" means queries of up to two words still require every word to + # match (75% of 2 rounds down to 1, which would match far too loosely); longer queries require 75%. + return Match(searchq_fuzzy={"query": query_parsed, "fuzziness": "AUTO", "minimum_should_match": "2<75%"}) + if search_settings.searchTypeSettings.fuzzySearch: + return Match(searchq_fuzzy={"query": query_parsed, "operator": "AND"}) + return Match(searchq_precise={"query": query_parsed, "operator": "AND"}) + + def get_search( search_settings: SearchSettings, query: str | None, card_types: list[CardType], expansion_code: str | None = None, collector_number: str | None = None, + fallback_level: int = 0, ) -> CardSearch: """ This is the core search function for MPC Autofill - queries Elasticsearch for `self` given `search_settings` and returns the list of corresponding `Card` identifiers. Expects that the search index exists. Since this function is called many times, it makes sense to check this once at the call site rather than in the body of this function. + `fallback_level` selects the match clause via `get_match` - level 0 is the standard search, and levels 1 and 2 + (see `FUZZY_FALLBACK_LEVELS`) are progressively more forgiving retries for when stricter levels match nothing. """ # set up search - match the query and use the AND operator @@ -124,11 +146,9 @@ def get_search( ) if query: query_parsed = to_searchable(query) - if search_settings.searchTypeSettings.fuzzySearch: - match = Match(searchq_fuzzy={"query": query_parsed, "operator": "AND"}) - else: - match = Match(searchq_precise={"query": query_parsed, "operator": "AND"}) - s = s.query(match) + s = s.query( + get_match(search_settings=search_settings, query_parsed=query_parsed, fallback_level=fallback_level) + ) if card_types: s = s.filter( Bool( @@ -162,20 +182,37 @@ def retrieve_card_identifiers( expansion_code: str | None = None, collector_number: str | None = None, ) -> list[str]: - hits_iterable = ( - get_search( - search_settings=search_settings, - query=query, - card_types=[card_type], - expansion_code=expansion_code, - collector_number=collector_number, - ) - .sort({"priority": {"order": "desc"}}) - .params(preserve_order=True) - .scan() - ) source_order = get_source_order(search_settings=search_settings) - return [result.identifier for result in sorted(hits_iterable, key=lambda result: source_order[result.source_pk])] + + def execute(fallback_level: int) -> list[str]: + hits_iterable = ( + get_search( + search_settings=search_settings, + query=query, + card_types=[card_type], + expansion_code=expansion_code, + collector_number=collector_number, + fallback_level=fallback_level, + ) + .sort({"priority": {"order": "desc"}}) + .params(preserve_order=True) + .scan() + ) + return [ + result.identifier for result in sorted(hits_iterable, key=lambda result: source_order[result.source_pk]) + ] + + identifiers = execute(fallback_level=0) + if query: + for fallback_level in FUZZY_FALLBACK_LEVELS: + if identifiers: + break + try: + identifiers = execute(fallback_level=fallback_level) + except ElasticTransportError: + # the fallback is best-effort: surface the primary search's (empty) results rather than an error + break + return identifiers def retrieve_cardback_identifiers(search_settings: SearchSettings) -> list[str]: @@ -236,6 +273,8 @@ def get_new_cards_paginator(source: Source) -> Paginator[QuerySet[Card]]: "get_elasticsearch_connection", "ping_elasticsearch", "elastic_connection", + "FUZZY_FALLBACK_LEVELS", + "get_match", "get_search", "retrieve_card_identifiers", "retrieve_cardback_identifiers", diff --git a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr index ac2ac5559..e94342b1a 100644 --- a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr +++ b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr @@ -193,7 +193,7 @@ 'CARD': dict({ 'Brainstorm': dict({ 'canonicalArtist': dict({ - 'name': 'Artist 102', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -416,7 +416,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 107', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -702,7 +702,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 109', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -1783,6 +1783,17 @@ 'status_code': 200, }) # --- +# name: TestPostEditorSearchResults.test_fuzzy_fallback_respects_filters + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + ]), + }), + }), + 'status_code': 200, + }) +# --- # name: TestPostEditorSearchResults.test_fuzzy_search dict({ 'json': dict({ @@ -1796,6 +1807,19 @@ 'status_code': 200, }) # --- +# name: TestPostEditorSearchResults.test_fuzzy_search_with_typo_falls_back + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + '1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1', + '1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU', + ]), + }), + }), + 'status_code': 200, + }) +# --- # name: TestPostEditorSearchResults.test_get_multiple_rows_filtered_includes_one_tag dict({ 'json': dict({ @@ -1965,6 +1989,31 @@ 'status_code': 400, }) # --- +# name: TestPostEditorSearchResults.test_precise_search_with_partial_name_falls_back + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + '1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1', + '1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU', + ]), + }), + }), + 'status_code': 200, + }) +# --- +# name: TestPostEditorSearchResults.test_precise_search_with_typo_falls_back_to_fuzzy + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + '1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5', + ]), + }), + }), + 'status_code': 200, + }) +# --- # name: TestPostEditorSearchResults.test_priority_ordering_in_search_results dict({ 'json': dict({ @@ -2305,6 +2354,19 @@ 'status_code': 200, }) # --- +# name: TestPostEditorSearchResults.test_search_with_extra_words_falls_back + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + '1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1', + '1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU', + ]), + }), + }), + 'status_code': 200, + }) +# --- # name: TestPostExploreSearchResults.test_explore_search[filter to cardback + token] dict({ 'json': dict({ @@ -2399,7 +2461,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 67', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -2673,7 +2735,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 66', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -3025,7 +3087,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 68', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, diff --git a/MPCAutofill/cardpicker/tests/conftest.py b/MPCAutofill/cardpicker/tests/conftest.py index f401eb86d..ac495bf89 100644 --- a/MPCAutofill/cardpicker/tests/conftest.py +++ b/MPCAutofill/cardpicker/tests/conftest.py @@ -1,7 +1,9 @@ import datetime as dt +import inspect import uuid from typing import Type +import factory as factory_boy import pytest from pytest_elasticsearch import factories from testcontainers.elasticsearch import ElasticSearchContainer @@ -12,6 +14,7 @@ from cardpicker.integrations.game.base import GameIntegration from cardpicker.models import Card, CardTypes, DFCPair, Source, Tag +from cardpicker.tests import factories as factories_module from cardpicker.tests.constants import Cards, DummyIntegration, Sources from cardpicker.tests.factories import ( CanonicalCardFactory, @@ -26,6 +29,18 @@ ELASTICSEARCH_PORT = 9300 # this is the default expected by `elasticsearch_nooproc` +@pytest.fixture(autouse=True) +def reset_factory_sequences() -> None: + """ + Snapshots embed sequence-generated values (e.g. "Artist 0"), so sequences must restart for every test - + otherwise adding or running a subset of tests shifts the numbering and breaks unrelated snapshots. + """ + + for _, factory_class in inspect.getmembers(factories_module, inspect.isclass): + if issubclass(factory_class, factory_boy.django.DjangoModelFactory): + factory_class.reset_sequence(0) + + @pytest.fixture(scope="session") def postgres_container(): postgres = PostgresContainer("postgres:16.0-alpine").with_bind_ports(5432, POSTGRES_PORT) diff --git a/MPCAutofill/cardpicker/tests/test_integrations.py b/MPCAutofill/cardpicker/tests/test_integrations.py index f206ae780..2287b8825 100644 --- a/MPCAutofill/cardpicker/tests/test_integrations.py +++ b/MPCAutofill/cardpicker/tests/test_integrations.py @@ -107,6 +107,8 @@ def test_get_meld_pairs(self): @pytest.mark.parametrize("url", [item.value for item in Decks], ids=[item.name.lower() for item in Decks]) def test_valid_url(self, client, django_settings, snapshot, url: str): + if "moxfield" in url and not conf_settings.MOXFIELD_SECRET: + pytest.skip("MOXFIELD_SECRET is not configured (e.g. fork PRs and environments without credentials)") decklist = MTGIntegration.query_import_site(url) assert decklist assert Counter(decklist.splitlines()) == snapshot diff --git a/MPCAutofill/cardpicker/tests/test_sources.py b/MPCAutofill/cardpicker/tests/test_sources.py index a686c61c6..948490a8c 100644 --- a/MPCAutofill/cardpicker/tests/test_sources.py +++ b/MPCAutofill/cardpicker/tests/test_sources.py @@ -1,4 +1,6 @@ import datetime as dt +import json +from pathlib import Path import freezegun import pytest @@ -21,6 +23,16 @@ DEFAULT_DATE = dt.datetime(2023, 1, 1) +def google_drive_credentials_available() -> bool: + # mirrors the path resolution in `cardpicker.sources.api`. fork PRs and local runs without + # credentials get a missing or unparseable client_secrets.json, so those tests must skip. + try: + with open(Path(__file__).parent.parent.parent / "client_secrets.json") as f: + return bool(json.load(f)) + except (OSError, json.JSONDecodeError): + return False + + class TestAPI: # region constants @@ -354,6 +366,10 @@ def test_unpack_name( # endregion +@pytest.mark.skipif( + not google_drive_credentials_available(), + reason="Google Drive credentials (client_secrets.json) are not available", +) class TestUpdateDatabase: # region tests diff --git a/MPCAutofill/cardpicker/tests/test_views.py b/MPCAutofill/cardpicker/tests/test_views.py index b2c16ccac..8261ae61c 100644 --- a/MPCAutofill/cardpicker/tests/test_views.py +++ b/MPCAutofill/cardpicker/tests/test_views.py @@ -205,6 +205,84 @@ def test_fuzzy_search(self, client, snapshot): Cards.PAST_IN_FLAMES_2.value.identifier, ] + def test_precise_search_with_typo_falls_back_to_fuzzy(self, client, snapshot): + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": BASE_SEARCH_SETTINGS, + "queries": {"key1": {"query": "brainstrom", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [Cards.BRAINSTORM.value.identifier] + + def test_fuzzy_search_with_typo_falls_back(self, client, snapshot): + search_settings = deepcopy(BASE_SEARCH_SETTINGS) + search_settings["searchTypeSettings"]["fuzzySearch"] = True + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": search_settings, + "queries": {"key1": {"query": "past in flams", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [ + Cards.PAST_IN_FLAMES_1.value.identifier, + Cards.PAST_IN_FLAMES_2.value.identifier, + ] + + def test_search_with_extra_words_falls_back(self, client, snapshot): + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": BASE_SEARCH_SETTINGS, + "queries": {"key1": {"query": "the past in flames", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [ + Cards.PAST_IN_FLAMES_1.value.identifier, + Cards.PAST_IN_FLAMES_2.value.identifier, + ] + + def test_precise_search_with_partial_name_falls_back(self, client, snapshot): + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": BASE_SEARCH_SETTINGS, + "queries": {"key1": {"query": "past in", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [ + Cards.PAST_IN_FLAMES_1.value.identifier, + Cards.PAST_IN_FLAMES_2.value.identifier, + ] + + def test_fuzzy_fallback_respects_filters(self, client, snapshot): + search_settings = deepcopy(BASE_SEARCH_SETTINGS) + search_settings["filterSettings"]["minimumDPI"] = 700 # excludes every card in the database + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": search_settings, + "queries": {"key1": {"query": "brainstrom", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [] + def test_minimum_dpi_yielding_no_search_results(self, client, snapshot): search_settings = deepcopy(BASE_SEARCH_SETTINGS) search_settings["filterSettings"]["minimumDPI"] = 400 @@ -643,6 +721,25 @@ def test_explore_search(self, client, snapshot, query, card_types, sort_by, expe expected_card.identifier for expected_card in expected_cards ] + def test_explore_search_with_typo_falls_back(self, client): + response = client.post( + reverse(views.post_explore_search), + { + "searchSettings": BASE_SEARCH_SETTINGS, + "query": "brainstrom", + "cardTypes": ["CARD"], + "sortBy": "dateCreatedDescending", + "pageSize": 20, + "pageStart": 0, + }, + content_type="application/json", + ) + # no snapshot here: serialised cards embed factory-sequenced artist names, which depend on test order + assert response.status_code == 200 + response_json = response.json() + assert response_json["count"] == 1 + assert [item["identifier"] for item in response_json["cards"]] == [Cards.BRAINSTORM.value.identifier] + @pytest.mark.parametrize( "page_start, page_size, expected_cards", [ diff --git a/MPCAutofill/cardpicker/views.py b/MPCAutofill/cardpicker/views.py index 7cb83f630..6f994bd13 100644 --- a/MPCAutofill/cardpicker/views.py +++ b/MPCAutofill/cardpicker/views.py @@ -5,6 +5,8 @@ from typing import Any, Callable, TypeVar, Union, cast import pycountry +from elasticsearch.exceptions import TransportError as ElasticTransportError +from elasticsearch_dsl import Search from elasticsearch_dsl.index import Index from pydantic import ValidationError @@ -58,6 +60,7 @@ TagsResponse, ) from cardpicker.search.search_functions import ( + FUZZY_FALLBACK_LEVELS, SearchExceptions, get_new_cards_paginator, get_search, @@ -207,12 +210,29 @@ def post_explore_search(request: HttpRequest) -> HttpResponse: SortBy.dateModifiedDescending: {"date_modified": {"order": "desc"}, "searchq_keyword": {"order": "asc"}}, }[explore_search_request.sortBy] - s = get_search( - search_settings=explore_search_request.searchSettings, - query=explore_search_request.query, - card_types=explore_search_request.cardTypes, - ).sort(sort) + def get_sorted_search(fallback_level: int) -> Search: + return get_search( + search_settings=explore_search_request.searchSettings, + query=explore_search_request.query, + card_types=explore_search_request.cardTypes, + fallback_level=fallback_level, + ).sort(sort) + + s = get_sorted_search(fallback_level=0) count = s.extra(track_total_hits=True).count() + if count == 0 and explore_search_request.query: + # the escalation loop is duplicated from `retrieve_card_identifiers` because explore search needs + # elasticsearch-side sorting and pagination rather than a materialised list of identifiers + try: + for fallback_level in FUZZY_FALLBACK_LEVELS: + fallback_s = get_sorted_search(fallback_level=fallback_level) + fallback_count = fallback_s.extra(track_total_hits=True).count() + if fallback_count > 0: + s, count = fallback_s, fallback_count + break + except ElasticTransportError: + # the fallback is best-effort: surface the primary search's (empty) results rather than an error + pass s_sliced = s[explore_search_request.pageStart : explore_search_request.pageStart + explore_search_request.pageSize] card_ids = [man.identifier for man in s_sliced.execute()] diff --git a/MPCAutofill/pytest.ini b/MPCAutofill/pytest.ini index 9f62bfe06..b86f91aef 100644 --- a/MPCAutofill/pytest.ini +++ b/MPCAutofill/pytest.ini @@ -1,2 +1,5 @@ [pytest] DJANGO_SETTINGS_MODULE = MPCAutofill.settings +# credential-dependent tests (update_database, Moxfield) skip when secrets are absent - e.g. on fork +# PRs - which leaves their committed snapshots unused; warn rather than fail the run in that case +addopts = --snapshot-warn-unused diff --git a/docs/superpowers/specs/2026-07-31-fuzzy-search-design.md b/docs/superpowers/specs/2026-07-31-fuzzy-search-design.md new file mode 100644 index 000000000..8125ccec9 --- /dev/null +++ b/docs/superpowers/specs/2026-07-31-fuzzy-search-design.md @@ -0,0 +1,92 @@ +# Fuzzy Search Fallback — Design + +**Date:** 2026-07-31 +**Status:** Approved pending review + +## Problem + +Card search fails outright when the query doesn't exactly match a card name. The +existing "Fuzzy (Forgiving) Search" toggle (default: off/precise) only switches +between whole-string matching and word-level matching with a hard AND — there is +no typo tolerance anywhere, and extra words (e.g. "the lightning bolt") cause +zero results even in forgiving mode. + +Failure kinds in scope: typos/misspellings, partial names, punctuation/accents, +word order/extra words. (Punctuation, accents, and word order are already mostly +handled by `to_searchable` sanitization and the ES analyzers; typos, extra +words, and precise-mode misses are the gaps this design closes.) + +## Chosen approach: retry-on-miss ("exact first, fuzzy fallback") + +Existing queries are untouched. When a search returns **zero** hits and a query +string is present, retry in a more forgiving mode (one retry on the client; +up to two escalating retries on the backend). If the primary +search finds one or more results, behavior is byte-for-byte identical to today. +The fallback fires in both precise and forgiving modes (the toggle defaults to +precise, so a fuzzy-mode-only fallback would leave the default experience +broken). + +Rejected alternatives: +- **Blended single query with boosting** — fuzzy candidates always pollute + results; hard to keep the two stacks consistent. +- **Index-time n-gram/phonetic fields** — requires ES schema migration + full + reindex, has no client-side (Orama) equivalent, overkill for card-name-length + strings. + +## Components + +### Client search (Orama) — `frontend/src/features/clientSearch/clientSearchService.worker.ts` + +`searchOramaIndex` is the single shared search routine (both `editorSearch` and +`exploreSearch` call it). Add the retry there: + +- Condition: initial search returns 0 hits AND `query` is defined/non-empty. +- Retry parameters: `exact: false`, `tolerance: 2` (Orama Levenshtein edit + distance). +- All `where` filters (card type, source, tags, DPI, size, printings, artists) + remain applied on the retry — fuzzy rescues bad spelling, never bypasses + filters. + +### Backend (Elasticsearch) — `MPCAutofill/cardpicker/search/search_functions.py` + +The match clause builder gets fallback variants, executed from the shared +retrieval function (so editor and explore endpoints both benefit) only when the +primary search returns 0 hits: + +1. First retry: `Match(searchq_fuzzy={"query": q, "operator": "AND", + "fuzziness": "AUTO"})` — typo tolerance, all words still required. +2. Second retry (only if the first retry also returns 0): + same, with `minimum_should_match: "75%"` replacing the AND operator — + tolerates extra words. + +All existing filters (source, DPI, size, card type, language, tags) remain on +the fallback queries. No index mapping changes, no reindex. + +## Data flow + +Unchanged. No API schema changes; request/response shapes are untouched. The +fallback is invisible to callers except that previously-empty result sets may +now contain matches. + +## Error handling + +The fallback is best-effort: if a retry throws (ES timeout, worker error), we +return the original empty result set rather than surfacing a new error path. + +## Testing + +- **Client (Jest, worker tests):** + - typo query: "lightnig bolt" → finds "Lightning Bolt" + - extra words: "the lightning bolt" → finds "Lightning Bolt" + - partial: "bolt" → finds "Lightning Bolt" + - guard: a query with exact hits returns identical results with the feature + in place + - filters still apply during fallback (e.g. excluded tag stays excluded) +- **Backend (pytest against real ES, existing `cardpicker/tests/` patterns):** + same five cases. + +## Dev infrastructure + +Bring up Postgres + Elasticsearch + Django locally via the repo's `docker/` +compose files, seeded with a small test drive, to exercise the backend +end-to-end. diff --git a/frontend/src/features/clientSearch/clientSearchService.worker.ts b/frontend/src/features/clientSearch/clientSearchService.worker.ts index f8ce317cf..4dc275797 100644 --- a/frontend/src/features/clientSearch/clientSearchService.worker.ts +++ b/frontend/src/features/clientSearch/clientSearchService.worker.ts @@ -1,8 +1,8 @@ -import { create, getByID, insertMultiple, search } from "@orama/orama"; +import { create, getByID, insertMultiple } from "@orama/orama"; import { expose } from "comlink"; import { Printing, Unknown } from "@/common/constants"; -import { computeSearchQueryHashKey, toSearchable } from "@/common/processing"; +import { computeSearchQueryHashKey } from "@/common/processing"; import { CardType as CardTypeSchema, SearchQuery, @@ -21,7 +21,6 @@ import { OramaCardDocument, OramaIndex, OramaSchema, - OramaSearchResult, OramaSearchResults, SearchResults, } from "@/common/types"; @@ -29,6 +28,7 @@ import { parseDjangoDate } from "@/common/utils"; import { getDefaultSearchSettings } from "@/store/slices/searchSettingsSlice"; import { Folder, GoogleDriveIndexer, LocalFilesIndexer } from "./indexer"; +import { searchOramaIndices } from "./oramaSearch"; export class ClientSearchService { private localFilesIndex: LocalFilesIndex | undefined; @@ -140,118 +140,6 @@ export class ClientSearchService { }; } - private searchOramaIndex( - oramaIndex: OramaIndex | undefined, - searchSettings: SearchSettings, - query: string | undefined, - cardTypes: Array, - sortBy?: SortBy, - limit?: number, - offset?: number, - printings?: Array, - artists?: Array - ): OramaSearchResults | undefined { - if (oramaIndex?.oramaDb === undefined) { - return undefined; - } - - const includesTags = searchSettings.filterSettings.includesTags.length > 0; - const excludesTags = searchSettings.filterSettings.excludesTags.length > 0; - - const sortByConfigs = { - [SortBy.DateCreatedAscending]: { - property: "createdNumber", - order: "ASC", - }, - [SortBy.DateCreatedDescending]: { - property: "createdNumber", - order: "DESC", - }, - [SortBy.DateModifiedAscending]: { - property: "lastModifiedNumber", - order: "ASC", - }, - [SortBy.DateModifiedDescending]: { - property: "lastModifiedNumber", - order: "DESC", - }, - [SortBy.NameAscending]: { property: "searchq", order: "ASC" }, - [SortBy.NameDescending]: { property: "searchq", order: "DESC" }, - } as const; - const sortByConfig = sortBy && sortByConfigs[sortBy]; - - const searchResults = search(oramaIndex.oramaDb, { - term: query ? toSearchable(query) : undefined, - properties: ["searchq"], - limit: limit ?? 1_000_000, // some arbitrary upper limit. if undefined, orama limits to 10 results. - offset: offset ?? 0, - exact: - query !== undefined && !searchSettings.searchTypeSettings.fuzzySearch, - where: { - and: [ - ...(cardTypes.length > 0 ? [{ cardType: { in: cardTypes } }] : []), - { - or: [ - ...searchSettings.sourceSettings.sources - .filter((sourceRow) => sourceRow[1] === true) - .map((sourceRow) => ({ sourceId: { eq: sourceRow[0] } })), - { sourceId: { eq: -1 } }, - ], - }, - ...(includesTags - ? [ - { - tags: { - containsAny: searchSettings.filterSettings.includesTags, - }, - }, - ] - : []), - ...(excludesTags - ? [ - { - not: { - tags: { - containsAny: searchSettings.filterSettings.excludesTags, - }, - }, - }, - ] - : []), - { - dpi: { - between: [ - searchSettings.filterSettings.minimumDPI, - searchSettings.filterSettings.maximumDPI, - ], - }, - }, - { - size: { - lte: searchSettings.filterSettings.maximumSize * 1_000_000, - }, - }, - ...((printings?.length ?? 0) > 0 - ? [ - { - or: printings!.map(({ expansionCode, collectorNumber }) => ({ - expansionCode: expansionCode, - collectorNumber: collectorNumber, - })), - }, - ] - : []), - ...((artists?.length ?? 0) > 0 ? [{ artist: { in: artists } }] : []), - ], - }, - sortBy: sortByConfig, - }) as { - hits: Array | undefined; - count: number | undefined; - }; - return { hits: searchResults.hits ?? [], count: searchResults.count ?? 0 }; - } - private search( searchSettings: SearchSettings, query: string | undefined, @@ -260,29 +148,14 @@ export class ClientSearchService { limit?: number, offset?: number ): OramaSearchResults | undefined { - return [this.localFilesIndex?.index, this.googleDriveIndex?.index].reduce( - ( - accumulated: OramaSearchResults, - index: OramaIndex | undefined - ): OramaSearchResults => { - if (index === undefined) { - return accumulated; - } - const searchResults = this.searchOramaIndex( - index, - searchSettings, - query, - cardTypes, - sortBy, - limit, - offset - ); - return { - hits: accumulated.hits.concat(searchResults?.hits ?? []), - count: accumulated.count + (searchResults?.count ?? 0), - }; - }, - { hits: [], count: 0 } + return searchOramaIndices( + [this.localFilesIndex?.index, this.googleDriveIndex?.index], + searchSettings, + query, + cardTypes, + sortBy, + limit, + offset ); } @@ -342,8 +215,8 @@ export class ClientSearchService { ); const oramaIndex: OramaIndex = { oramaDb, size: cards.length }; - const results = this.searchOramaIndex( - oramaIndex, + const results = searchOramaIndices( + [oramaIndex], searchSettings, undefined, [], @@ -354,10 +227,10 @@ export class ClientSearchService { artists ); if (sortBy !== undefined) { - return results?.hits.map((hit) => hit.id) ?? []; + return results.hits.map((hit) => hit.id); } else { // honour the ordering of `cards` - const resultsSet = new Set(results?.hits.map((hit) => hit.id)); + const resultsSet = new Set(results.hits.map((hit) => hit.id)); return cards .map((card) => card.identifier) .filter((identifier) => resultsSet.has(identifier)); diff --git a/frontend/src/features/clientSearch/oramaSearch.test.ts b/frontend/src/features/clientSearch/oramaSearch.test.ts new file mode 100644 index 000000000..2bb34c80a --- /dev/null +++ b/frontend/src/features/clientSearch/oramaSearch.test.ts @@ -0,0 +1,176 @@ +import { create, insertMultiple } from "@orama/orama"; + +import { toSearchable } from "@/common/processing"; +import { CardType as CardTypeSchema, SourceType } from "@/common/schema_types"; +import { + OramaCardDocument, + OramaIndex, + OramaSchema, + SearchSettings, +} from "@/common/types"; +import { getDefaultSearchSettings } from "@/store/slices/searchSettingsSlice"; + +import { searchOramaIndices } from "./oramaSearch"; + +const buildCardDocument = ( + name: string, + overrides: Partial = {} +): OramaCardDocument => ({ + id: name, + name, + searchq: toSearchable(name), + source: "test-source", + sourceId: -1, + sourceVerbose: "Test Source", + cardType: CardTypeSchema.Card, + extension: "png", + language: "EN", + tags: [], + dpi: 600, + size: 1_000_000, + lastModified: new Date(2020, 0, 1), + lastModifiedNumber: new Date(2020, 0, 1).valueOf(), + created: new Date(2020, 0, 1), + createdNumber: new Date(2020, 0, 1).valueOf(), + expansionCode: "UNK", + collectorNumber: "UNK", + artist: "Test Artist", + params: { + sourceType: SourceType.GoogleDrive, + identifier: name, + fileHandle: undefined, + }, + ...overrides, +}); + +const buildIndex = async ( + documents: Array +): Promise => { + const oramaDb = await create({ + schema: OramaSchema, + sort: { + enabled: true, + unsortableProperties: [ + "id", + "name", + "cardType", + "extension", + "language", + "tags", + "dpi", + "size", + ], + }, + }); + await insertMultiple(oramaDb, documents); + return { oramaDb, size: documents.length }; +}; + +const preciseSettings = (): SearchSettings => + getDefaultSearchSettings({}, false); + +const searchNames = ( + indices: OramaIndex | Array, + settings: SearchSettings, + query: string +): Array => + searchOramaIndices( + Array.isArray(indices) ? indices : [indices], + settings, + query, + [CardTypeSchema.Card] + ).hits.map((hit) => hit.document.name); + +describe("searchOramaIndex fuzzy fallback", () => { + let index: OramaIndex; + + beforeEach(async () => { + index = await buildIndex([ + buildCardDocument("Lightning Bolt"), + buildCardDocument("Lightning Helix"), + buildCardDocument("Counterspell"), + ]); + }); + + test("finds card despite a typo in every word", async () => { + expect(searchNames(index, preciseSettings(), "lightnig bol")).toContain( + "Lightning Bolt" + ); + }); + + test("finds card despite a typo in a single-word query", async () => { + expect(searchNames(index, preciseSettings(), "counterspel")).toContain( + "Counterspell" + ); + }); + + test("finds card when query contains extra words", async () => { + expect( + searchNames(index, preciseSettings(), "the lightning bolt card") + ).toContain("Lightning Bolt"); + }); + + test("finds card from a partial name", async () => { + expect(searchNames(index, preciseSettings(), "bolt")).toContain( + "Lightning Bolt" + ); + }); + + test("exact query returns only the exactly-matching card", async () => { + expect(searchNames(index, preciseSettings(), "lightning bolt")).toEqual([ + "Lightning Bolt", + ]); + }); + + test("fallback does not bypass tag exclusion filters", async () => { + const settings = preciseSettings(); + // default settings exclude the NSFW tag + const taggedIndex = await buildIndex([ + buildCardDocument("Lightning Bolt", { tags: ["NSFW"] }), + ]); + expect(searchNames(taggedIndex, settings, "lightnig bol")).toEqual([]); + }); + + test("fallback does not bypass DPI filters", async () => { + const lowDpiIndex = await buildIndex([ + buildCardDocument("Lightning Bolt", { dpi: 100 }), + ]); + const settings = preciseSettings(); + settings.filterSettings.minimumDPI = 300; + expect(searchNames(lowDpiIndex, settings, "lightnig bol")).toEqual([]); + }); + + test("returns empty results when no indices are defined", () => { + expect( + searchOramaIndices([undefined], preciseSettings(), "bolt", [ + CardTypeSchema.Card, + ]) + ).toEqual({ hits: [], count: 0 }); + }); + + describe("multiple indices", () => { + let indexA: OramaIndex; + let indexB: OramaIndex; + + beforeEach(async () => { + indexA = await buildIndex([buildCardDocument("Lightning Bolt")]); + indexB = await buildIndex([buildCardDocument("Lightning Belt")]); + }); + + test("an exact hit in one index suppresses fuzzy matches from the others", async () => { + expect( + searchNames([indexA, indexB], preciseSettings(), "lightning bolt") + ).toEqual(["Lightning Bolt"]); + }); + + test("fallback fires across all indices when none has an exact hit", async () => { + const names = searchNames( + [indexA, indexB], + preciseSettings(), + "lightnig bol" + ); + expect(names).toContain("Lightning Bolt"); + expect(names).toContain("Lightning Belt"); + }); + }); +}); diff --git a/frontend/src/features/clientSearch/oramaSearch.ts b/frontend/src/features/clientSearch/oramaSearch.ts new file mode 100644 index 000000000..ba3916059 --- /dev/null +++ b/frontend/src/features/clientSearch/oramaSearch.ts @@ -0,0 +1,195 @@ +import { search } from "@orama/orama"; + +import { Printing } from "@/common/constants"; +import { toSearchable } from "@/common/processing"; +import { SearchSettings, SortBy } from "@/common/schema_types"; +import { + CardType, + OramaIndex, + OramaSearchResult, + OramaSearchResults, +} from "@/common/types"; + +// Levenshtein edit distance applied per search term when the initial search +// finds nothing. 2 forgives most single-word misspellings without letting +// unrelated cards through. +const FALLBACK_TOLERANCE = 2; + +function searchSingleIndex( + searchOptions: { exact: boolean; tolerance?: number }, + oramaIndex: OramaIndex | undefined, + searchSettings: SearchSettings, + query: string | undefined, + cardTypes: Array, + sortBy?: SortBy, + limit?: number, + offset?: number, + printings?: Array, + artists?: Array +): OramaSearchResults | undefined { + if (oramaIndex?.oramaDb === undefined) { + return undefined; + } + + const includesTags = searchSettings.filterSettings.includesTags.length > 0; + const excludesTags = searchSettings.filterSettings.excludesTags.length > 0; + + const sortByConfigs = { + [SortBy.DateCreatedAscending]: { + property: "createdNumber", + order: "ASC", + }, + [SortBy.DateCreatedDescending]: { + property: "createdNumber", + order: "DESC", + }, + [SortBy.DateModifiedAscending]: { + property: "lastModifiedNumber", + order: "ASC", + }, + [SortBy.DateModifiedDescending]: { + property: "lastModifiedNumber", + order: "DESC", + }, + [SortBy.NameAscending]: { property: "searchq", order: "ASC" }, + [SortBy.NameDescending]: { property: "searchq", order: "DESC" }, + } as const; + const sortByConfig = sortBy && sortByConfigs[sortBy]; + + const runSearch = (options: { + exact: boolean; + tolerance?: number; + }): OramaSearchResults => { + const searchResults = search(oramaIndex.oramaDb, { + term: query ? toSearchable(query) : undefined, + properties: ["searchq"], + limit: limit ?? 1_000_000, // some arbitrary upper limit. if undefined, orama limits to 10 results. + offset: offset ?? 0, + exact: options.exact, + tolerance: options.tolerance, + where: { + and: [ + ...(cardTypes.length > 0 ? [{ cardType: { in: cardTypes } }] : []), + { + or: [ + ...searchSettings.sourceSettings.sources + .filter((sourceRow) => sourceRow[1] === true) + .map((sourceRow) => ({ sourceId: { eq: sourceRow[0] } })), + { sourceId: { eq: -1 } }, + ], + }, + ...(includesTags + ? [ + { + tags: { + containsAny: searchSettings.filterSettings.includesTags, + }, + }, + ] + : []), + ...(excludesTags + ? [ + { + not: { + tags: { + containsAny: searchSettings.filterSettings.excludesTags, + }, + }, + }, + ] + : []), + { + dpi: { + between: [ + searchSettings.filterSettings.minimumDPI, + searchSettings.filterSettings.maximumDPI, + ], + }, + }, + { + size: { + lte: searchSettings.filterSettings.maximumSize * 1_000_000, + }, + }, + ...((printings?.length ?? 0) > 0 + ? [ + { + or: printings!.map(({ expansionCode, collectorNumber }) => ({ + expansionCode: expansionCode, + collectorNumber: collectorNumber, + })), + }, + ] + : []), + ...((artists?.length ?? 0) > 0 ? [{ artist: { in: artists } }] : []), + ], + }, + sortBy: sortByConfig, + }) as { + hits: Array | undefined; + count: number | undefined; + }; + return { + hits: searchResults.hits ?? [], + count: searchResults.count ?? 0, + }; + }; + + return runSearch(searchOptions); +} + +export function searchOramaIndices( + oramaIndices: Array, + searchSettings: SearchSettings, + query: string | undefined, + cardTypes: Array, + sortBy?: SortBy, + limit?: number, + offset?: number, + printings?: Array, + artists?: Array +): OramaSearchResults { + const runSearches = (searchOptions: { + exact: boolean; + tolerance?: number; + }): OramaSearchResults => + oramaIndices.reduce( + ( + accumulated: OramaSearchResults, + oramaIndex: OramaIndex | undefined + ): OramaSearchResults => { + const searchResults = searchSingleIndex( + searchOptions, + oramaIndex, + searchSettings, + query, + cardTypes, + sortBy, + limit, + offset, + printings, + artists + ); + return { + hits: accumulated.hits.concat(searchResults?.hits ?? []), + count: accumulated.count + (searchResults?.count ?? 0), + }; + }, + { hits: [], count: 0 } + ); + + const primaryResults = runSearches({ + exact: + query !== undefined && !searchSettings.searchTypeSettings.fuzzySearch, + }); + if (primaryResults.count > 0 || !query) { + return primaryResults; + } + // fuzzy fallback: the query found nothing in any index, so retry once with + // typo tolerance. all `where` filters still apply. + try { + return runSearches({ exact: false, tolerance: FALLBACK_TOLERANCE }); + } catch { + return primaryResults; + } +}