From bb4aade211bbd88585d325f98d6dd6a01273d075 Mon Sep 17 00:00:00 2001 From: GH Date: Fri, 31 Jul 2026 22:43:00 +0000 Subject: [PATCH 1/5] Add fuzzy search fallback design spec Co-Authored-By: Claude Fable 5 --- .../specs/2026-07-31-fuzzy-search-design.md | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-31-fuzzy-search-design.md diff --git a/docs/superpowers/specs/2026-07-31-fuzzy-search-design.md b/docs/superpowers/specs/2026-07-31-fuzzy-search-design.md new file mode 100644 index 000000000..8125ccec9 --- /dev/null +++ b/docs/superpowers/specs/2026-07-31-fuzzy-search-design.md @@ -0,0 +1,92 @@ +# Fuzzy Search Fallback — Design + +**Date:** 2026-07-31 +**Status:** Approved pending review + +## Problem + +Card search fails outright when the query doesn't exactly match a card name. The +existing "Fuzzy (Forgiving) Search" toggle (default: off/precise) only switches +between whole-string matching and word-level matching with a hard AND — there is +no typo tolerance anywhere, and extra words (e.g. "the lightning bolt") cause +zero results even in forgiving mode. + +Failure kinds in scope: typos/misspellings, partial names, punctuation/accents, +word order/extra words. (Punctuation, accents, and word order are already mostly +handled by `to_searchable` sanitization and the ES analyzers; typos, extra +words, and precise-mode misses are the gaps this design closes.) + +## Chosen approach: retry-on-miss ("exact first, fuzzy fallback") + +Existing queries are untouched. When a search returns **zero** hits and a query +string is present, retry in a more forgiving mode (one retry on the client; +up to two escalating retries on the backend). If the primary +search finds one or more results, behavior is byte-for-byte identical to today. +The fallback fires in both precise and forgiving modes (the toggle defaults to +precise, so a fuzzy-mode-only fallback would leave the default experience +broken). + +Rejected alternatives: +- **Blended single query with boosting** — fuzzy candidates always pollute + results; hard to keep the two stacks consistent. +- **Index-time n-gram/phonetic fields** — requires ES schema migration + full + reindex, has no client-side (Orama) equivalent, overkill for card-name-length + strings. + +## Components + +### Client search (Orama) — `frontend/src/features/clientSearch/clientSearchService.worker.ts` + +`searchOramaIndex` is the single shared search routine (both `editorSearch` and +`exploreSearch` call it). Add the retry there: + +- Condition: initial search returns 0 hits AND `query` is defined/non-empty. +- Retry parameters: `exact: false`, `tolerance: 2` (Orama Levenshtein edit + distance). +- All `where` filters (card type, source, tags, DPI, size, printings, artists) + remain applied on the retry — fuzzy rescues bad spelling, never bypasses + filters. + +### Backend (Elasticsearch) — `MPCAutofill/cardpicker/search/search_functions.py` + +The match clause builder gets fallback variants, executed from the shared +retrieval function (so editor and explore endpoints both benefit) only when the +primary search returns 0 hits: + +1. First retry: `Match(searchq_fuzzy={"query": q, "operator": "AND", + "fuzziness": "AUTO"})` — typo tolerance, all words still required. +2. Second retry (only if the first retry also returns 0): + same, with `minimum_should_match: "75%"` replacing the AND operator — + tolerates extra words. + +All existing filters (source, DPI, size, card type, language, tags) remain on +the fallback queries. No index mapping changes, no reindex. + +## Data flow + +Unchanged. No API schema changes; request/response shapes are untouched. The +fallback is invisible to callers except that previously-empty result sets may +now contain matches. + +## Error handling + +The fallback is best-effort: if a retry throws (ES timeout, worker error), we +return the original empty result set rather than surfacing a new error path. + +## Testing + +- **Client (Jest, worker tests):** + - typo query: "lightnig bolt" → finds "Lightning Bolt" + - extra words: "the lightning bolt" → finds "Lightning Bolt" + - partial: "bolt" → finds "Lightning Bolt" + - guard: a query with exact hits returns identical results with the feature + in place + - filters still apply during fallback (e.g. excluded tag stays excluded) +- **Backend (pytest against real ES, existing `cardpicker/tests/` patterns):** + same five cases. + +## Dev infrastructure + +Bring up Postgres + Elasticsearch + Django locally via the repo's `docker/` +compose files, seeded with a small test drive, to exercise the backend +end-to-end. From e1e4b130ace832351e1a75d52e5a6db26748eea1 Mon Sep 17 00:00:00 2001 From: GH Date: Fri, 31 Jul 2026 22:54:21 +0000 Subject: [PATCH 2/5] Add fuzzy fallback to client-side card search When a search returns zero hits, retry once with Levenshtein tolerance so typos, extra words, and near-misses still find cards. Extracts searchOramaIndex into its own module so the search logic is testable outside the worker. Co-Authored-By: Claude Fable 5 --- .../clientSearchService.worker.ts | 196 ++++-------------- .../features/clientSearch/oramaSearch.test.ts | 147 +++++++++++++ .../src/features/clientSearch/oramaSearch.ts | 151 ++++++++++++++ 3 files changed, 341 insertions(+), 153 deletions(-) create mode 100644 frontend/src/features/clientSearch/oramaSearch.test.ts create mode 100644 frontend/src/features/clientSearch/oramaSearch.ts diff --git a/frontend/src/features/clientSearch/clientSearchService.worker.ts b/frontend/src/features/clientSearch/clientSearchService.worker.ts index f8ce317cf..03a7dcc9f 100644 --- a/frontend/src/features/clientSearch/clientSearchService.worker.ts +++ b/frontend/src/features/clientSearch/clientSearchService.worker.ts @@ -1,8 +1,8 @@ -import { create, getByID, insertMultiple, search } from "@orama/orama"; +import { create, getByID, insertMultiple } from "@orama/orama"; import { expose } from "comlink"; import { Printing, Unknown } from "@/common/constants"; -import { computeSearchQueryHashKey, toSearchable } from "@/common/processing"; +import { computeSearchQueryHashKey } from "@/common/processing"; import { CardType as CardTypeSchema, SearchQuery, @@ -21,7 +21,6 @@ import { OramaCardDocument, OramaIndex, OramaSchema, - OramaSearchResult, OramaSearchResults, SearchResults, } from "@/common/types"; @@ -29,6 +28,7 @@ import { parseDjangoDate } from "@/common/utils"; import { getDefaultSearchSettings } from "@/store/slices/searchSettingsSlice"; import { Folder, GoogleDriveIndexer, LocalFilesIndexer } from "./indexer"; +import { searchOramaIndex } from "./oramaSearch"; export class ClientSearchService { private localFilesIndex: LocalFilesIndex | undefined; @@ -53,7 +53,7 @@ export class ClientSearchService { public async setLocalFilesDirectoryHandle( directoryHandle: FileSystemDirectoryHandle, - tags: Array | undefined + tags: Array | undefined, ) { this.localFilesIndex = { fileHandle: directoryHandle, @@ -78,7 +78,7 @@ export class ClientSearchService { } public async indexDirectory( - tags: Array | undefined + tags: Array | undefined, ): Promise<{ handle: FileSystemDirectoryHandle; size: number } | undefined> { if (this.localFilesIndex?.fileHandle !== undefined) { const oramaIndex = await new LocalFilesIndexer().indexFiles( @@ -90,11 +90,11 @@ export class ClientSearchService { sourceType: SourceType.LocalFile, }, this.localFilesIndex.fileHandle.name, - undefined + undefined, ), ], [], - tags + tags, ); this.localFilesIndex.index = oramaIndex; return { @@ -109,7 +109,7 @@ export class ClientSearchService { tags: Array | undefined, bearerToken: string, folders: Array, - images: Array + images: Array, ) { const indexer = new GoogleDriveIndexer(bearerToken); const oramaIndex = await indexer.indexFiles( @@ -122,17 +122,17 @@ export class ClientSearchService { fileHandle: undefined, }, name, - undefined - ) + undefined, + ), ), ( await Promise.all( images.map(async ({ id }) => - indexer.getImageFromIdentifier(id, undefined) - ) + indexer.getImageFromIdentifier(id, undefined), + ), ) ).filter((image) => image !== undefined), - tags + tags, ); this.googleDriveIndex = { index: oramaIndex }; return { @@ -140,149 +140,37 @@ export class ClientSearchService { }; } - private searchOramaIndex( - oramaIndex: OramaIndex | undefined, - searchSettings: SearchSettings, - query: string | undefined, - cardTypes: Array, - sortBy?: SortBy, - limit?: number, - offset?: number, - printings?: Array, - artists?: Array - ): OramaSearchResults | undefined { - if (oramaIndex?.oramaDb === undefined) { - return undefined; - } - - const includesTags = searchSettings.filterSettings.includesTags.length > 0; - const excludesTags = searchSettings.filterSettings.excludesTags.length > 0; - - const sortByConfigs = { - [SortBy.DateCreatedAscending]: { - property: "createdNumber", - order: "ASC", - }, - [SortBy.DateCreatedDescending]: { - property: "createdNumber", - order: "DESC", - }, - [SortBy.DateModifiedAscending]: { - property: "lastModifiedNumber", - order: "ASC", - }, - [SortBy.DateModifiedDescending]: { - property: "lastModifiedNumber", - order: "DESC", - }, - [SortBy.NameAscending]: { property: "searchq", order: "ASC" }, - [SortBy.NameDescending]: { property: "searchq", order: "DESC" }, - } as const; - const sortByConfig = sortBy && sortByConfigs[sortBy]; - - const searchResults = search(oramaIndex.oramaDb, { - term: query ? toSearchable(query) : undefined, - properties: ["searchq"], - limit: limit ?? 1_000_000, // some arbitrary upper limit. if undefined, orama limits to 10 results. - offset: offset ?? 0, - exact: - query !== undefined && !searchSettings.searchTypeSettings.fuzzySearch, - where: { - and: [ - ...(cardTypes.length > 0 ? [{ cardType: { in: cardTypes } }] : []), - { - or: [ - ...searchSettings.sourceSettings.sources - .filter((sourceRow) => sourceRow[1] === true) - .map((sourceRow) => ({ sourceId: { eq: sourceRow[0] } })), - { sourceId: { eq: -1 } }, - ], - }, - ...(includesTags - ? [ - { - tags: { - containsAny: searchSettings.filterSettings.includesTags, - }, - }, - ] - : []), - ...(excludesTags - ? [ - { - not: { - tags: { - containsAny: searchSettings.filterSettings.excludesTags, - }, - }, - }, - ] - : []), - { - dpi: { - between: [ - searchSettings.filterSettings.minimumDPI, - searchSettings.filterSettings.maximumDPI, - ], - }, - }, - { - size: { - lte: searchSettings.filterSettings.maximumSize * 1_000_000, - }, - }, - ...((printings?.length ?? 0) > 0 - ? [ - { - or: printings!.map(({ expansionCode, collectorNumber }) => ({ - expansionCode: expansionCode, - collectorNumber: collectorNumber, - })), - }, - ] - : []), - ...((artists?.length ?? 0) > 0 ? [{ artist: { in: artists } }] : []), - ], - }, - sortBy: sortByConfig, - }) as { - hits: Array | undefined; - count: number | undefined; - }; - return { hits: searchResults.hits ?? [], count: searchResults.count ?? 0 }; - } - private search( searchSettings: SearchSettings, query: string | undefined, cardTypes: Array, sortBy?: SortBy, limit?: number, - offset?: number + offset?: number, ): OramaSearchResults | undefined { return [this.localFilesIndex?.index, this.googleDriveIndex?.index].reduce( ( accumulated: OramaSearchResults, - index: OramaIndex | undefined + index: OramaIndex | undefined, ): OramaSearchResults => { if (index === undefined) { return accumulated; } - const searchResults = this.searchOramaIndex( + const searchResults = searchOramaIndex( index, searchSettings, query, cardTypes, sortBy, limit, - offset + offset, ); return { hits: accumulated.hits.concat(searchResults?.hits ?? []), count: accumulated.count + (searchResults?.count ?? 0), }; }, - { hits: [], count: 0 } + { hits: [], count: 0 }, ); } @@ -291,7 +179,7 @@ export class ClientSearchService { searchSettings: SearchSettings, sortBy: SortBy | undefined, artists: Array, - printings: Array + printings: Array, ): Promise> { const oramaDb = await create({ schema: OramaSchema, @@ -337,12 +225,12 @@ export class ClientSearchService { expansionCode: card.canonicalCard?.expansionCode ?? Unknown, collectorNumber: card.canonicalCard?.collectorNumber ?? Unknown, artist: card.canonicalArtist?.name ?? Unknown, - }) - ) + }), + ), ); const oramaIndex: OramaIndex = { oramaDb, size: cards.length }; - const results = this.searchOramaIndex( + const results = searchOramaIndex( oramaIndex, searchSettings, undefined, @@ -351,7 +239,7 @@ export class ClientSearchService { undefined, undefined, printings, - artists + artists, ); if (sortBy !== undefined) { return results?.hits.map((hit) => hit.id) ?? []; @@ -369,7 +257,7 @@ export class ClientSearchService { query: string | undefined, cardTypes: Array, limit?: number, - offset?: number + offset?: number, ): Array | undefined { const results = this.search( searchSettings, @@ -377,7 +265,7 @@ export class ClientSearchService { cardTypes, undefined, limit, - offset + offset, ); return results !== undefined ? results.hits.map((cardDocument) => cardDocument.id) @@ -386,7 +274,7 @@ export class ClientSearchService { public editorSearch( searchSettings: SearchSettings, - searchQueries: Array + searchQueries: Array, ): SearchResults { const localResults: SearchResults = {}; for (const searchQuery of searchQueries) { @@ -399,7 +287,7 @@ export class ClientSearchService { const localResultsForQuery = this.retrieveCardIdentifiers( searchSettings, searchQuery.query, - [searchQuery.cardType] + [searchQuery.cardType], ); if (localResultsForQuery !== undefined) { localResults[hashkey] = localResultsForQuery; @@ -415,7 +303,7 @@ export class ClientSearchService { cardTypes: Array, searchSettings: SearchSettings, pageStart: number, - pageSize: number + pageSize: number, ): { cards: Array; count: number } { const searchResults = this.search( searchSettings, @@ -423,7 +311,7 @@ export class ClientSearchService { cardTypes, sortBy, pageSize, - pageStart + pageStart, ); const cardIds = searchResults?.hits?.map(({ id }) => id) ?? []; const cards = this.getCardDocumentsArray(cardIds); @@ -434,14 +322,14 @@ export class ClientSearchService { } public retrieveCardbackIdentifiers( - searchSettings: SearchSettings + searchSettings: SearchSettings, ): Array | undefined { return this.retrieveCardIdentifiers( searchSettings.searchTypeSettings.filterCardbacks ? searchSettings : getDefaultSearchSettings([], false), undefined, - [CardTypeSchema.Cardback] + [CardTypeSchema.Cardback], ); } @@ -453,7 +341,7 @@ export class ClientSearchService { } private getCardDocumentsArray( - identifiersToSearch: Array + identifiersToSearch: Array, ): Array { return identifiersToSearch.reduce( (accumulated: Array, identifier: string) => { @@ -463,7 +351,7 @@ export class ClientSearchService { } return accumulated; }, - [] as Array + [] as Array, ); } @@ -477,8 +365,8 @@ export class ClientSearchService { } return accumulated; }, - [] as Array<[string, CardDocument]> - ) + [] as Array<[string, CardDocument]>, + ), ); } @@ -490,7 +378,7 @@ export class ClientSearchService { if (oramaDb) { const result: OramaCardDocument | undefined = getByID( oramaDb, - identifier + identifier, ); if (result) { return result; @@ -501,7 +389,7 @@ export class ClientSearchService { } public translateOramaCardDocumentToCardDocument( - oramaCardDocument: OramaCardDocument + oramaCardDocument: OramaCardDocument, ): CardDocument { const lastModified = oramaCardDocument.lastModified.toLocaleDateString( undefined, @@ -510,7 +398,7 @@ export class ClientSearchService { year: "numeric", month: "long", day: "numeric", - } + }, ); return { identifier: oramaCardDocument.id, @@ -553,12 +441,14 @@ export class ClientSearchService { undefined, [cardType], undefined, - cardType === CardTypeSchema.Card ? 4 : 1 + cardType === CardTypeSchema.Card ? 4 : 1, )?.hits?.map((result) => - this.translateOramaCardDocumentToCardDocument(result.document) + this.translateOramaCardDocumentToCardDocument( + result.document, + ), ) : [], - ]) + ]), ) as { [cardType in CardType]: Array }) : undefined; } diff --git a/frontend/src/features/clientSearch/oramaSearch.test.ts b/frontend/src/features/clientSearch/oramaSearch.test.ts new file mode 100644 index 000000000..8c412dfb5 --- /dev/null +++ b/frontend/src/features/clientSearch/oramaSearch.test.ts @@ -0,0 +1,147 @@ +import { create, insertMultiple } from "@orama/orama"; + +import { toSearchable } from "@/common/processing"; +import { CardType as CardTypeSchema, SourceType } from "@/common/schema_types"; +import { + OramaCardDocument, + OramaIndex, + OramaSchema, + SearchSettings, +} from "@/common/types"; +import { getDefaultSearchSettings } from "@/store/slices/searchSettingsSlice"; + +import { searchOramaIndex } from "./oramaSearch"; + +const buildCardDocument = ( + name: string, + overrides: Partial = {} +): OramaCardDocument => ({ + id: name, + name, + searchq: toSearchable(name), + source: "test-source", + sourceId: -1, + sourceVerbose: "Test Source", + cardType: CardTypeSchema.Card, + extension: "png", + language: "EN", + tags: [], + dpi: 600, + size: 1_000_000, + lastModified: new Date(2020, 0, 1), + lastModifiedNumber: new Date(2020, 0, 1).valueOf(), + created: new Date(2020, 0, 1), + createdNumber: new Date(2020, 0, 1).valueOf(), + expansionCode: "UNK", + collectorNumber: "UNK", + artist: "Test Artist", + params: { + sourceType: SourceType.GoogleDrive, + identifier: name, + fileHandle: undefined, + }, + ...overrides, +}); + +const buildIndex = async ( + documents: Array +): Promise => { + const oramaDb = await create({ + schema: OramaSchema, + sort: { + enabled: true, + unsortableProperties: [ + "id", + "name", + "cardType", + "extension", + "language", + "tags", + "dpi", + "size", + ], + }, + }); + await insertMultiple(oramaDb, documents); + return { oramaDb, size: documents.length }; +}; + +const preciseSettings = (): SearchSettings => + getDefaultSearchSettings({}, false); + +const searchNames = ( + index: OramaIndex, + settings: SearchSettings, + query: string +): Array => + ( + searchOramaIndex(index, settings, query, [CardTypeSchema.Card])?.hits ?? [] + ).map((hit) => hit.document.name); + +describe("searchOramaIndex fuzzy fallback", () => { + let index: OramaIndex; + + beforeEach(async () => { + index = await buildIndex([ + buildCardDocument("Lightning Bolt"), + buildCardDocument("Lightning Helix"), + buildCardDocument("Counterspell"), + ]); + }); + + test("finds card despite a typo in every word", async () => { + expect(searchNames(index, preciseSettings(), "lightnig bol")).toContain( + "Lightning Bolt" + ); + }); + + test("finds card despite a typo in a single-word query", async () => { + expect(searchNames(index, preciseSettings(), "counterspel")).toContain( + "Counterspell" + ); + }); + + test("finds card when query contains extra words", async () => { + expect( + searchNames(index, preciseSettings(), "the lightning bolt card") + ).toContain("Lightning Bolt"); + }); + + test("finds card from a partial name", async () => { + expect(searchNames(index, preciseSettings(), "bolt")).toContain( + "Lightning Bolt" + ); + }); + + test("exact query returns only the exactly-matching card", async () => { + expect(searchNames(index, preciseSettings(), "lightning bolt")).toEqual([ + "Lightning Bolt", + ]); + }); + + test("fallback does not bypass tag exclusion filters", async () => { + const settings = preciseSettings(); + // default settings exclude the NSFW tag + const taggedIndex = await buildIndex([ + buildCardDocument("Lightning Bolt", { tags: ["NSFW"] }), + ]); + expect(searchNames(taggedIndex, settings, "lightnig bol")).toEqual([]); + }); + + test("fallback does not bypass DPI filters", async () => { + const lowDpiIndex = await buildIndex([ + buildCardDocument("Lightning Bolt", { dpi: 100 }), + ]); + const settings = preciseSettings(); + settings.filterSettings.minimumDPI = 300; + expect(searchNames(lowDpiIndex, settings, "lightnig bol")).toEqual([]); + }); + + test("returns undefined when the index is undefined", () => { + expect( + searchOramaIndex(undefined, preciseSettings(), "bolt", [ + CardTypeSchema.Card, + ]) + ).toBeUndefined(); + }); +}); diff --git a/frontend/src/features/clientSearch/oramaSearch.ts b/frontend/src/features/clientSearch/oramaSearch.ts new file mode 100644 index 000000000..3efa3708a --- /dev/null +++ b/frontend/src/features/clientSearch/oramaSearch.ts @@ -0,0 +1,151 @@ +import { search } from "@orama/orama"; + +import { Printing } from "@/common/constants"; +import { toSearchable } from "@/common/processing"; +import { SearchSettings, SortBy } from "@/common/schema_types"; +import { + CardType, + OramaIndex, + OramaSearchResult, + OramaSearchResults, +} from "@/common/types"; + +// Levenshtein edit distance applied per search term when the initial search +// finds nothing. 2 forgives most single-word misspellings without letting +// unrelated cards through. +const FALLBACK_TOLERANCE = 2; + +export function searchOramaIndex( + oramaIndex: OramaIndex | undefined, + searchSettings: SearchSettings, + query: string | undefined, + cardTypes: Array, + sortBy?: SortBy, + limit?: number, + offset?: number, + printings?: Array, + artists?: Array, +): OramaSearchResults | undefined { + if (oramaIndex?.oramaDb === undefined) { + return undefined; + } + + const includesTags = searchSettings.filterSettings.includesTags.length > 0; + const excludesTags = searchSettings.filterSettings.excludesTags.length > 0; + + const sortByConfigs = { + [SortBy.DateCreatedAscending]: { + property: "createdNumber", + order: "ASC", + }, + [SortBy.DateCreatedDescending]: { + property: "createdNumber", + order: "DESC", + }, + [SortBy.DateModifiedAscending]: { + property: "lastModifiedNumber", + order: "ASC", + }, + [SortBy.DateModifiedDescending]: { + property: "lastModifiedNumber", + order: "DESC", + }, + [SortBy.NameAscending]: { property: "searchq", order: "ASC" }, + [SortBy.NameDescending]: { property: "searchq", order: "DESC" }, + } as const; + const sortByConfig = sortBy && sortByConfigs[sortBy]; + + const runSearch = (options: { + exact: boolean; + tolerance?: number; + }): OramaSearchResults => { + const searchResults = search(oramaIndex.oramaDb, { + term: query ? toSearchable(query) : undefined, + properties: ["searchq"], + limit: limit ?? 1_000_000, // some arbitrary upper limit. if undefined, orama limits to 10 results. + offset: offset ?? 0, + exact: options.exact, + tolerance: options.tolerance, + where: { + and: [ + ...(cardTypes.length > 0 ? [{ cardType: { in: cardTypes } }] : []), + { + or: [ + ...searchSettings.sourceSettings.sources + .filter((sourceRow) => sourceRow[1] === true) + .map((sourceRow) => ({ sourceId: { eq: sourceRow[0] } })), + { sourceId: { eq: -1 } }, + ], + }, + ...(includesTags + ? [ + { + tags: { + containsAny: searchSettings.filterSettings.includesTags, + }, + }, + ] + : []), + ...(excludesTags + ? [ + { + not: { + tags: { + containsAny: searchSettings.filterSettings.excludesTags, + }, + }, + }, + ] + : []), + { + dpi: { + between: [ + searchSettings.filterSettings.minimumDPI, + searchSettings.filterSettings.maximumDPI, + ], + }, + }, + { + size: { + lte: searchSettings.filterSettings.maximumSize * 1_000_000, + }, + }, + ...((printings?.length ?? 0) > 0 + ? [ + { + or: printings!.map(({ expansionCode, collectorNumber }) => ({ + expansionCode: expansionCode, + collectorNumber: collectorNumber, + })), + }, + ] + : []), + ...((artists?.length ?? 0) > 0 ? [{ artist: { in: artists } }] : []), + ], + }, + sortBy: sortByConfig, + }) as { + hits: Array | undefined; + count: number | undefined; + }; + return { + hits: searchResults.hits ?? [], + count: searchResults.count ?? 0, + }; + }; + + const primaryResults = runSearch({ + exact: + query !== undefined && !searchSettings.searchTypeSettings.fuzzySearch, + }); + if (primaryResults.count > 0 || !query) { + return primaryResults; + } + // fuzzy fallback: the query found nothing, so retry once with typo + // tolerance. all `where` filters still apply. + try { + return runSearch({ exact: false, tolerance: FALLBACK_TOLERANCE }); + } catch { + return primaryResults; + } +} From ecae4d2f6089de1f8c7e0a762fe0921e90e31107 Mon Sep 17 00:00:00 2001 From: GH Date: Fri, 31 Jul 2026 23:13:53 +0000 Subject: [PATCH 3/5] Add fuzzy fallback to backend card search When a search matches nothing, retry with escalating forgiveness: first with typo tolerance (fuzziness AUTO, all words required), then also tolerating extra words (minimum_should_match 75%). Applies to both the editor search and explore search endpoints. Filters always apply to fallback queries, and no index mapping changes are needed. Co-Authored-By: Claude Fable 5 --- .../cardpicker/search/search_functions.py | 71 ++++++++++---- .../tests/__snapshots__/test_views.ambr | 62 ++++++++++++ MPCAutofill/cardpicker/tests/test_views.py | 97 +++++++++++++++++++ MPCAutofill/cardpicker/views.py | 22 ++++- 4 files changed, 229 insertions(+), 23 deletions(-) diff --git a/MPCAutofill/cardpicker/search/search_functions.py b/MPCAutofill/cardpicker/search/search_functions.py index a2baf12a3..1a4720549 100644 --- a/MPCAutofill/cardpicker/search/search_functions.py +++ b/MPCAutofill/cardpicker/search/search_functions.py @@ -88,12 +88,30 @@ def get_scaled_maximum_size(search_settings: SearchSettings) -> int: return search_settings.filterSettings.maximumSize * 1_000_000 +# fallback levels for the fuzzy retry-on-miss behaviour. level 0 is the standard match; levels 1 and 2 are +# progressively more forgiving and are only queried when every stricter level matched nothing. +FUZZY_FALLBACK_LEVELS = [1, 2] + + +def get_match(search_settings: SearchSettings, query_parsed: str, fallback_level: int = 0) -> Match: + if fallback_level == 1: + # forgive typos (up to elasticsearch's automatic edit distance), but still require every word to match + return Match(searchq_fuzzy={"query": query_parsed, "operator": "AND", "fuzziness": "AUTO"}) + if fallback_level == 2: + # forgive typos and extra words + return Match(searchq_fuzzy={"query": query_parsed, "fuzziness": "AUTO", "minimum_should_match": "75%"}) + if search_settings.searchTypeSettings.fuzzySearch: + return Match(searchq_fuzzy={"query": query_parsed, "operator": "AND"}) + return Match(searchq_precise={"query": query_parsed, "operator": "AND"}) + + def get_search( search_settings: SearchSettings, query: str | None, card_types: list[CardType], expansion_code: str | None = None, collector_number: str | None = None, + fallback_level: int = 0, ) -> CardSearch: """ This is the core search function for MPC Autofill - queries Elasticsearch for `self` given `search_settings` @@ -124,11 +142,9 @@ def get_search( ) if query: query_parsed = to_searchable(query) - if search_settings.searchTypeSettings.fuzzySearch: - match = Match(searchq_fuzzy={"query": query_parsed, "operator": "AND"}) - else: - match = Match(searchq_precise={"query": query_parsed, "operator": "AND"}) - s = s.query(match) + s = s.query( + get_match(search_settings=search_settings, query_parsed=query_parsed, fallback_level=fallback_level) + ) if card_types: s = s.filter( Bool( @@ -162,20 +178,37 @@ def retrieve_card_identifiers( expansion_code: str | None = None, collector_number: str | None = None, ) -> list[str]: - hits_iterable = ( - get_search( - search_settings=search_settings, - query=query, - card_types=[card_type], - expansion_code=expansion_code, - collector_number=collector_number, - ) - .sort({"priority": {"order": "desc"}}) - .params(preserve_order=True) - .scan() - ) source_order = get_source_order(search_settings=search_settings) - return [result.identifier for result in sorted(hits_iterable, key=lambda result: source_order[result.source_pk])] + + def execute(fallback_level: int) -> list[str]: + hits_iterable = ( + get_search( + search_settings=search_settings, + query=query, + card_types=[card_type], + expansion_code=expansion_code, + collector_number=collector_number, + fallback_level=fallback_level, + ) + .sort({"priority": {"order": "desc"}}) + .params(preserve_order=True) + .scan() + ) + return [ + result.identifier for result in sorted(hits_iterable, key=lambda result: source_order[result.source_pk]) + ] + + identifiers = execute(fallback_level=0) + if query: + for fallback_level in FUZZY_FALLBACK_LEVELS: + if identifiers: + break + try: + identifiers = execute(fallback_level=fallback_level) + except ElasticConnectionError: + # the fallback is best-effort: surface the primary search's (empty) results rather than an error + break + return identifiers def retrieve_cardback_identifiers(search_settings: SearchSettings) -> list[str]: @@ -236,6 +269,8 @@ def get_new_cards_paginator(source: Source) -> Paginator[QuerySet[Card]]: "get_elasticsearch_connection", "ping_elasticsearch", "elastic_connection", + "FUZZY_FALLBACK_LEVELS", + "get_match", "get_search", "retrieve_card_identifiers", "retrieve_cardback_identifiers", diff --git a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr index ac2ac5559..4f2b04a30 100644 --- a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr +++ b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr @@ -1783,6 +1783,17 @@ 'status_code': 200, }) # --- +# name: TestPostEditorSearchResults.test_fuzzy_fallback_respects_filters + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + ]), + }), + }), + 'status_code': 200, + }) +# --- # name: TestPostEditorSearchResults.test_fuzzy_search dict({ 'json': dict({ @@ -1796,6 +1807,19 @@ 'status_code': 200, }) # --- +# name: TestPostEditorSearchResults.test_fuzzy_search_with_typo_falls_back + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + '1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1', + '1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU', + ]), + }), + }), + 'status_code': 200, + }) +# --- # name: TestPostEditorSearchResults.test_get_multiple_rows_filtered_includes_one_tag dict({ 'json': dict({ @@ -1965,6 +1989,31 @@ 'status_code': 400, }) # --- +# name: TestPostEditorSearchResults.test_precise_search_with_partial_name_falls_back + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + '1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1', + '1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU', + ]), + }), + }), + 'status_code': 200, + }) +# --- +# name: TestPostEditorSearchResults.test_precise_search_with_typo_falls_back_to_fuzzy + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + '1c4M-sK9gd0Xju0NXCPtqeTW_DQTldVU5', + ]), + }), + }), + 'status_code': 200, + }) +# --- # name: TestPostEditorSearchResults.test_priority_ordering_in_search_results dict({ 'json': dict({ @@ -2305,6 +2354,19 @@ 'status_code': 200, }) # --- +# name: TestPostEditorSearchResults.test_search_with_extra_words_falls_back + dict({ + 'json': dict({ + 'results': dict({ + 'key1': list([ + '1UPdh7J7hScg4ZnxSPJ-EeBYHLp2s3Oz1', + '1dxSLHtw-VwwE09pZCA8OA6LbuWRZPEoU', + ]), + }), + }), + 'status_code': 200, + }) +# --- # name: TestPostExploreSearchResults.test_explore_search[filter to cardback + token] dict({ 'json': dict({ diff --git a/MPCAutofill/cardpicker/tests/test_views.py b/MPCAutofill/cardpicker/tests/test_views.py index b2c16ccac..8261ae61c 100644 --- a/MPCAutofill/cardpicker/tests/test_views.py +++ b/MPCAutofill/cardpicker/tests/test_views.py @@ -205,6 +205,84 @@ def test_fuzzy_search(self, client, snapshot): Cards.PAST_IN_FLAMES_2.value.identifier, ] + def test_precise_search_with_typo_falls_back_to_fuzzy(self, client, snapshot): + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": BASE_SEARCH_SETTINGS, + "queries": {"key1": {"query": "brainstrom", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [Cards.BRAINSTORM.value.identifier] + + def test_fuzzy_search_with_typo_falls_back(self, client, snapshot): + search_settings = deepcopy(BASE_SEARCH_SETTINGS) + search_settings["searchTypeSettings"]["fuzzySearch"] = True + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": search_settings, + "queries": {"key1": {"query": "past in flams", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [ + Cards.PAST_IN_FLAMES_1.value.identifier, + Cards.PAST_IN_FLAMES_2.value.identifier, + ] + + def test_search_with_extra_words_falls_back(self, client, snapshot): + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": BASE_SEARCH_SETTINGS, + "queries": {"key1": {"query": "the past in flames", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [ + Cards.PAST_IN_FLAMES_1.value.identifier, + Cards.PAST_IN_FLAMES_2.value.identifier, + ] + + def test_precise_search_with_partial_name_falls_back(self, client, snapshot): + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": BASE_SEARCH_SETTINGS, + "queries": {"key1": {"query": "past in", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [ + Cards.PAST_IN_FLAMES_1.value.identifier, + Cards.PAST_IN_FLAMES_2.value.identifier, + ] + + def test_fuzzy_fallback_respects_filters(self, client, snapshot): + search_settings = deepcopy(BASE_SEARCH_SETTINGS) + search_settings["filterSettings"]["minimumDPI"] = 700 # excludes every card in the database + response = client.post( + reverse(views.post_editor_search), + { + "searchSettings": search_settings, + "queries": {"key1": {"query": "brainstrom", "cardType": "CARD"}}, + }, + content_type="application/json", + ) + snapshot_response(response, snapshot) + assert response.status_code == 200 + assert response.json()["results"]["key1"] == [] + def test_minimum_dpi_yielding_no_search_results(self, client, snapshot): search_settings = deepcopy(BASE_SEARCH_SETTINGS) search_settings["filterSettings"]["minimumDPI"] = 400 @@ -643,6 +721,25 @@ def test_explore_search(self, client, snapshot, query, card_types, sort_by, expe expected_card.identifier for expected_card in expected_cards ] + def test_explore_search_with_typo_falls_back(self, client): + response = client.post( + reverse(views.post_explore_search), + { + "searchSettings": BASE_SEARCH_SETTINGS, + "query": "brainstrom", + "cardTypes": ["CARD"], + "sortBy": "dateCreatedDescending", + "pageSize": 20, + "pageStart": 0, + }, + content_type="application/json", + ) + # no snapshot here: serialised cards embed factory-sequenced artist names, which depend on test order + assert response.status_code == 200 + response_json = response.json() + assert response_json["count"] == 1 + assert [item["identifier"] for item in response_json["cards"]] == [Cards.BRAINSTORM.value.identifier] + @pytest.mark.parametrize( "page_start, page_size, expected_cards", [ diff --git a/MPCAutofill/cardpicker/views.py b/MPCAutofill/cardpicker/views.py index 7cb83f630..be7888baf 100644 --- a/MPCAutofill/cardpicker/views.py +++ b/MPCAutofill/cardpicker/views.py @@ -58,6 +58,7 @@ TagsResponse, ) from cardpicker.search.search_functions import ( + FUZZY_FALLBACK_LEVELS, SearchExceptions, get_new_cards_paginator, get_search, @@ -207,12 +208,23 @@ def post_explore_search(request: HttpRequest) -> HttpResponse: SortBy.dateModifiedDescending: {"date_modified": {"order": "desc"}, "searchq_keyword": {"order": "asc"}}, }[explore_search_request.sortBy] - s = get_search( - search_settings=explore_search_request.searchSettings, - query=explore_search_request.query, - card_types=explore_search_request.cardTypes, - ).sort(sort) + def get_sorted_search(fallback_level: int) -> Any: + return get_search( + search_settings=explore_search_request.searchSettings, + query=explore_search_request.query, + card_types=explore_search_request.cardTypes, + fallback_level=fallback_level, + ).sort(sort) + + s = get_sorted_search(fallback_level=0) count = s.extra(track_total_hits=True).count() + if count == 0 and explore_search_request.query: + for fallback_level in FUZZY_FALLBACK_LEVELS: + fallback_s = get_sorted_search(fallback_level=fallback_level) + fallback_count = fallback_s.extra(track_total_hits=True).count() + if fallback_count > 0: + s, count = fallback_s, fallback_count + break s_sliced = s[explore_search_request.pageStart : explore_search_request.pageStart + explore_search_request.pageSize] card_ids = [man.identifier for man in s_sliced.execute()] From 8a6be096395dd00ec9f739069c25e857e10f86f5 Mon Sep 17 00:00:00 2001 From: GH Date: Fri, 31 Jul 2026 23:26:57 +0000 Subject: [PATCH 4/5] Apply code review fixes to fuzzy search fallback - Reformat with prettier 2.7.1 to match the repo's pinned pre-commit hook - Hoist the client-side fallback decision from per-index to aggregate level so an exact hit in one index suppresses fuzzy matches from others (new searchOramaIndices API, with multi-index tests) - Make the explore view's fallback best-effort like the editor path - Use minimum_should_match 2<75% so one- and two-word queries still require every word during the extra-words fallback - Broaden fallback retry catch to TransportError - Type the explore search closure and document fallback_level Co-Authored-By: Claude Fable 5 --- .../cardpicker/search/search_functions.py | 10 +- MPCAutofill/cardpicker/views.py | 22 ++-- .../clientSearchService.worker.ts | 109 ++++++++---------- .../features/clientSearch/oramaSearch.test.ts | 45 ++++++-- .../src/features/clientSearch/oramaSearch.ts | 56 ++++++++- 5 files changed, 155 insertions(+), 87 deletions(-) diff --git a/MPCAutofill/cardpicker/search/search_functions.py b/MPCAutofill/cardpicker/search/search_functions.py index 1a4720549..a6726a7dd 100644 --- a/MPCAutofill/cardpicker/search/search_functions.py +++ b/MPCAutofill/cardpicker/search/search_functions.py @@ -5,6 +5,7 @@ import pycountry from elasticsearch import Elasticsearch from elasticsearch.exceptions import ConnectionError as ElasticConnectionError +from elasticsearch.exceptions import TransportError as ElasticTransportError from elasticsearch_dsl.query import Bool, Match, Range, Terms from django.conf import settings @@ -98,8 +99,9 @@ def get_match(search_settings: SearchSettings, query_parsed: str, fallback_level # forgive typos (up to elasticsearch's automatic edit distance), but still require every word to match return Match(searchq_fuzzy={"query": query_parsed, "operator": "AND", "fuzziness": "AUTO"}) if fallback_level == 2: - # forgive typos and extra words - return Match(searchq_fuzzy={"query": query_parsed, "fuzziness": "AUTO", "minimum_should_match": "75%"}) + # forgive typos and extra words. "2<75%" means queries of up to two words still require every word to + # match (75% of 2 rounds down to 1, which would match far too loosely); longer queries require 75%. + return Match(searchq_fuzzy={"query": query_parsed, "fuzziness": "AUTO", "minimum_should_match": "2<75%"}) if search_settings.searchTypeSettings.fuzzySearch: return Match(searchq_fuzzy={"query": query_parsed, "operator": "AND"}) return Match(searchq_precise={"query": query_parsed, "operator": "AND"}) @@ -118,6 +120,8 @@ def get_search( and returns the list of corresponding `Card` identifiers. Expects that the search index exists. Since this function is called many times, it makes sense to check this once at the call site rather than in the body of this function. + `fallback_level` selects the match clause via `get_match` - level 0 is the standard search, and levels 1 and 2 + (see `FUZZY_FALLBACK_LEVELS`) are progressively more forgiving retries for when stricter levels match nothing. """ # set up search - match the query and use the AND operator @@ -205,7 +209,7 @@ def execute(fallback_level: int) -> list[str]: break try: identifiers = execute(fallback_level=fallback_level) - except ElasticConnectionError: + except ElasticTransportError: # the fallback is best-effort: surface the primary search's (empty) results rather than an error break return identifiers diff --git a/MPCAutofill/cardpicker/views.py b/MPCAutofill/cardpicker/views.py index be7888baf..6f994bd13 100644 --- a/MPCAutofill/cardpicker/views.py +++ b/MPCAutofill/cardpicker/views.py @@ -5,6 +5,8 @@ from typing import Any, Callable, TypeVar, Union, cast import pycountry +from elasticsearch.exceptions import TransportError as ElasticTransportError +from elasticsearch_dsl import Search from elasticsearch_dsl.index import Index from pydantic import ValidationError @@ -208,7 +210,7 @@ def post_explore_search(request: HttpRequest) -> HttpResponse: SortBy.dateModifiedDescending: {"date_modified": {"order": "desc"}, "searchq_keyword": {"order": "asc"}}, }[explore_search_request.sortBy] - def get_sorted_search(fallback_level: int) -> Any: + def get_sorted_search(fallback_level: int) -> Search: return get_search( search_settings=explore_search_request.searchSettings, query=explore_search_request.query, @@ -219,12 +221,18 @@ def get_sorted_search(fallback_level: int) -> Any: s = get_sorted_search(fallback_level=0) count = s.extra(track_total_hits=True).count() if count == 0 and explore_search_request.query: - for fallback_level in FUZZY_FALLBACK_LEVELS: - fallback_s = get_sorted_search(fallback_level=fallback_level) - fallback_count = fallback_s.extra(track_total_hits=True).count() - if fallback_count > 0: - s, count = fallback_s, fallback_count - break + # the escalation loop is duplicated from `retrieve_card_identifiers` because explore search needs + # elasticsearch-side sorting and pagination rather than a materialised list of identifiers + try: + for fallback_level in FUZZY_FALLBACK_LEVELS: + fallback_s = get_sorted_search(fallback_level=fallback_level) + fallback_count = fallback_s.extra(track_total_hits=True).count() + if fallback_count > 0: + s, count = fallback_s, fallback_count + break + except ElasticTransportError: + # the fallback is best-effort: surface the primary search's (empty) results rather than an error + pass s_sliced = s[explore_search_request.pageStart : explore_search_request.pageStart + explore_search_request.pageSize] card_ids = [man.identifier for man in s_sliced.execute()] diff --git a/frontend/src/features/clientSearch/clientSearchService.worker.ts b/frontend/src/features/clientSearch/clientSearchService.worker.ts index 03a7dcc9f..4dc275797 100644 --- a/frontend/src/features/clientSearch/clientSearchService.worker.ts +++ b/frontend/src/features/clientSearch/clientSearchService.worker.ts @@ -28,7 +28,7 @@ import { parseDjangoDate } from "@/common/utils"; import { getDefaultSearchSettings } from "@/store/slices/searchSettingsSlice"; import { Folder, GoogleDriveIndexer, LocalFilesIndexer } from "./indexer"; -import { searchOramaIndex } from "./oramaSearch"; +import { searchOramaIndices } from "./oramaSearch"; export class ClientSearchService { private localFilesIndex: LocalFilesIndex | undefined; @@ -53,7 +53,7 @@ export class ClientSearchService { public async setLocalFilesDirectoryHandle( directoryHandle: FileSystemDirectoryHandle, - tags: Array | undefined, + tags: Array | undefined ) { this.localFilesIndex = { fileHandle: directoryHandle, @@ -78,7 +78,7 @@ export class ClientSearchService { } public async indexDirectory( - tags: Array | undefined, + tags: Array | undefined ): Promise<{ handle: FileSystemDirectoryHandle; size: number } | undefined> { if (this.localFilesIndex?.fileHandle !== undefined) { const oramaIndex = await new LocalFilesIndexer().indexFiles( @@ -90,11 +90,11 @@ export class ClientSearchService { sourceType: SourceType.LocalFile, }, this.localFilesIndex.fileHandle.name, - undefined, + undefined ), ], [], - tags, + tags ); this.localFilesIndex.index = oramaIndex; return { @@ -109,7 +109,7 @@ export class ClientSearchService { tags: Array | undefined, bearerToken: string, folders: Array, - images: Array, + images: Array ) { const indexer = new GoogleDriveIndexer(bearerToken); const oramaIndex = await indexer.indexFiles( @@ -122,17 +122,17 @@ export class ClientSearchService { fileHandle: undefined, }, name, - undefined, - ), + undefined + ) ), ( await Promise.all( images.map(async ({ id }) => - indexer.getImageFromIdentifier(id, undefined), - ), + indexer.getImageFromIdentifier(id, undefined) + ) ) ).filter((image) => image !== undefined), - tags, + tags ); this.googleDriveIndex = { index: oramaIndex }; return { @@ -146,31 +146,16 @@ export class ClientSearchService { cardTypes: Array, sortBy?: SortBy, limit?: number, - offset?: number, + offset?: number ): OramaSearchResults | undefined { - return [this.localFilesIndex?.index, this.googleDriveIndex?.index].reduce( - ( - accumulated: OramaSearchResults, - index: OramaIndex | undefined, - ): OramaSearchResults => { - if (index === undefined) { - return accumulated; - } - const searchResults = searchOramaIndex( - index, - searchSettings, - query, - cardTypes, - sortBy, - limit, - offset, - ); - return { - hits: accumulated.hits.concat(searchResults?.hits ?? []), - count: accumulated.count + (searchResults?.count ?? 0), - }; - }, - { hits: [], count: 0 }, + return searchOramaIndices( + [this.localFilesIndex?.index, this.googleDriveIndex?.index], + searchSettings, + query, + cardTypes, + sortBy, + limit, + offset ); } @@ -179,7 +164,7 @@ export class ClientSearchService { searchSettings: SearchSettings, sortBy: SortBy | undefined, artists: Array, - printings: Array, + printings: Array ): Promise> { const oramaDb = await create({ schema: OramaSchema, @@ -225,13 +210,13 @@ export class ClientSearchService { expansionCode: card.canonicalCard?.expansionCode ?? Unknown, collectorNumber: card.canonicalCard?.collectorNumber ?? Unknown, artist: card.canonicalArtist?.name ?? Unknown, - }), - ), + }) + ) ); const oramaIndex: OramaIndex = { oramaDb, size: cards.length }; - const results = searchOramaIndex( - oramaIndex, + const results = searchOramaIndices( + [oramaIndex], searchSettings, undefined, [], @@ -239,13 +224,13 @@ export class ClientSearchService { undefined, undefined, printings, - artists, + artists ); if (sortBy !== undefined) { - return results?.hits.map((hit) => hit.id) ?? []; + return results.hits.map((hit) => hit.id); } else { // honour the ordering of `cards` - const resultsSet = new Set(results?.hits.map((hit) => hit.id)); + const resultsSet = new Set(results.hits.map((hit) => hit.id)); return cards .map((card) => card.identifier) .filter((identifier) => resultsSet.has(identifier)); @@ -257,7 +242,7 @@ export class ClientSearchService { query: string | undefined, cardTypes: Array, limit?: number, - offset?: number, + offset?: number ): Array | undefined { const results = this.search( searchSettings, @@ -265,7 +250,7 @@ export class ClientSearchService { cardTypes, undefined, limit, - offset, + offset ); return results !== undefined ? results.hits.map((cardDocument) => cardDocument.id) @@ -274,7 +259,7 @@ export class ClientSearchService { public editorSearch( searchSettings: SearchSettings, - searchQueries: Array, + searchQueries: Array ): SearchResults { const localResults: SearchResults = {}; for (const searchQuery of searchQueries) { @@ -287,7 +272,7 @@ export class ClientSearchService { const localResultsForQuery = this.retrieveCardIdentifiers( searchSettings, searchQuery.query, - [searchQuery.cardType], + [searchQuery.cardType] ); if (localResultsForQuery !== undefined) { localResults[hashkey] = localResultsForQuery; @@ -303,7 +288,7 @@ export class ClientSearchService { cardTypes: Array, searchSettings: SearchSettings, pageStart: number, - pageSize: number, + pageSize: number ): { cards: Array; count: number } { const searchResults = this.search( searchSettings, @@ -311,7 +296,7 @@ export class ClientSearchService { cardTypes, sortBy, pageSize, - pageStart, + pageStart ); const cardIds = searchResults?.hits?.map(({ id }) => id) ?? []; const cards = this.getCardDocumentsArray(cardIds); @@ -322,14 +307,14 @@ export class ClientSearchService { } public retrieveCardbackIdentifiers( - searchSettings: SearchSettings, + searchSettings: SearchSettings ): Array | undefined { return this.retrieveCardIdentifiers( searchSettings.searchTypeSettings.filterCardbacks ? searchSettings : getDefaultSearchSettings([], false), undefined, - [CardTypeSchema.Cardback], + [CardTypeSchema.Cardback] ); } @@ -341,7 +326,7 @@ export class ClientSearchService { } private getCardDocumentsArray( - identifiersToSearch: Array, + identifiersToSearch: Array ): Array { return identifiersToSearch.reduce( (accumulated: Array, identifier: string) => { @@ -351,7 +336,7 @@ export class ClientSearchService { } return accumulated; }, - [] as Array, + [] as Array ); } @@ -365,8 +350,8 @@ export class ClientSearchService { } return accumulated; }, - [] as Array<[string, CardDocument]>, - ), + [] as Array<[string, CardDocument]> + ) ); } @@ -378,7 +363,7 @@ export class ClientSearchService { if (oramaDb) { const result: OramaCardDocument | undefined = getByID( oramaDb, - identifier, + identifier ); if (result) { return result; @@ -389,7 +374,7 @@ export class ClientSearchService { } public translateOramaCardDocumentToCardDocument( - oramaCardDocument: OramaCardDocument, + oramaCardDocument: OramaCardDocument ): CardDocument { const lastModified = oramaCardDocument.lastModified.toLocaleDateString( undefined, @@ -398,7 +383,7 @@ export class ClientSearchService { year: "numeric", month: "long", day: "numeric", - }, + } ); return { identifier: oramaCardDocument.id, @@ -441,14 +426,12 @@ export class ClientSearchService { undefined, [cardType], undefined, - cardType === CardTypeSchema.Card ? 4 : 1, + cardType === CardTypeSchema.Card ? 4 : 1 )?.hits?.map((result) => - this.translateOramaCardDocumentToCardDocument( - result.document, - ), + this.translateOramaCardDocumentToCardDocument(result.document) ) : [], - ]), + ]) ) as { [cardType in CardType]: Array }) : undefined; } diff --git a/frontend/src/features/clientSearch/oramaSearch.test.ts b/frontend/src/features/clientSearch/oramaSearch.test.ts index 8c412dfb5..2bb34c80a 100644 --- a/frontend/src/features/clientSearch/oramaSearch.test.ts +++ b/frontend/src/features/clientSearch/oramaSearch.test.ts @@ -10,7 +10,7 @@ import { } from "@/common/types"; import { getDefaultSearchSettings } from "@/store/slices/searchSettingsSlice"; -import { searchOramaIndex } from "./oramaSearch"; +import { searchOramaIndices } from "./oramaSearch"; const buildCardDocument = ( name: string, @@ -70,13 +70,16 @@ const preciseSettings = (): SearchSettings => getDefaultSearchSettings({}, false); const searchNames = ( - index: OramaIndex, + indices: OramaIndex | Array, settings: SearchSettings, query: string ): Array => - ( - searchOramaIndex(index, settings, query, [CardTypeSchema.Card])?.hits ?? [] - ).map((hit) => hit.document.name); + searchOramaIndices( + Array.isArray(indices) ? indices : [indices], + settings, + query, + [CardTypeSchema.Card] + ).hits.map((hit) => hit.document.name); describe("searchOramaIndex fuzzy fallback", () => { let index: OramaIndex; @@ -137,11 +140,37 @@ describe("searchOramaIndex fuzzy fallback", () => { expect(searchNames(lowDpiIndex, settings, "lightnig bol")).toEqual([]); }); - test("returns undefined when the index is undefined", () => { + test("returns empty results when no indices are defined", () => { expect( - searchOramaIndex(undefined, preciseSettings(), "bolt", [ + searchOramaIndices([undefined], preciseSettings(), "bolt", [ CardTypeSchema.Card, ]) - ).toBeUndefined(); + ).toEqual({ hits: [], count: 0 }); + }); + + describe("multiple indices", () => { + let indexA: OramaIndex; + let indexB: OramaIndex; + + beforeEach(async () => { + indexA = await buildIndex([buildCardDocument("Lightning Bolt")]); + indexB = await buildIndex([buildCardDocument("Lightning Belt")]); + }); + + test("an exact hit in one index suppresses fuzzy matches from the others", async () => { + expect( + searchNames([indexA, indexB], preciseSettings(), "lightning bolt") + ).toEqual(["Lightning Bolt"]); + }); + + test("fallback fires across all indices when none has an exact hit", async () => { + const names = searchNames( + [indexA, indexB], + preciseSettings(), + "lightnig bol" + ); + expect(names).toContain("Lightning Bolt"); + expect(names).toContain("Lightning Belt"); + }); }); }); diff --git a/frontend/src/features/clientSearch/oramaSearch.ts b/frontend/src/features/clientSearch/oramaSearch.ts index 3efa3708a..ba3916059 100644 --- a/frontend/src/features/clientSearch/oramaSearch.ts +++ b/frontend/src/features/clientSearch/oramaSearch.ts @@ -15,7 +15,8 @@ import { // unrelated cards through. const FALLBACK_TOLERANCE = 2; -export function searchOramaIndex( +function searchSingleIndex( + searchOptions: { exact: boolean; tolerance?: number }, oramaIndex: OramaIndex | undefined, searchSettings: SearchSettings, query: string | undefined, @@ -24,7 +25,7 @@ export function searchOramaIndex( limit?: number, offset?: number, printings?: Array, - artists?: Array, + artists?: Array ): OramaSearchResults | undefined { if (oramaIndex?.oramaDb === undefined) { return undefined; @@ -134,17 +135,60 @@ export function searchOramaIndex( }; }; - const primaryResults = runSearch({ + return runSearch(searchOptions); +} + +export function searchOramaIndices( + oramaIndices: Array, + searchSettings: SearchSettings, + query: string | undefined, + cardTypes: Array, + sortBy?: SortBy, + limit?: number, + offset?: number, + printings?: Array, + artists?: Array +): OramaSearchResults { + const runSearches = (searchOptions: { + exact: boolean; + tolerance?: number; + }): OramaSearchResults => + oramaIndices.reduce( + ( + accumulated: OramaSearchResults, + oramaIndex: OramaIndex | undefined + ): OramaSearchResults => { + const searchResults = searchSingleIndex( + searchOptions, + oramaIndex, + searchSettings, + query, + cardTypes, + sortBy, + limit, + offset, + printings, + artists + ); + return { + hits: accumulated.hits.concat(searchResults?.hits ?? []), + count: accumulated.count + (searchResults?.count ?? 0), + }; + }, + { hits: [], count: 0 } + ); + + const primaryResults = runSearches({ exact: query !== undefined && !searchSettings.searchTypeSettings.fuzzySearch, }); if (primaryResults.count > 0 || !query) { return primaryResults; } - // fuzzy fallback: the query found nothing, so retry once with typo - // tolerance. all `where` filters still apply. + // fuzzy fallback: the query found nothing in any index, so retry once with + // typo tolerance. all `where` filters still apply. try { - return runSearch({ exact: false, tolerance: FALLBACK_TOLERANCE }); + return runSearches({ exact: false, tolerance: FALLBACK_TOLERANCE }); } catch { return primaryResults; } From 2bad262a1bb738db87efd671414134445e63ba8d Mon Sep 17 00:00:00 2001 From: GH Date: Sat, 1 Aug 2026 04:28:58 +0000 Subject: [PATCH 5/5] Make backend test suite deterministic and credential-independent - Reset factory sequences before every test so snapshots no longer depend on which tests run or in what order. Previously, adding any test shifted sequence-generated values (e.g. artist names) in every later test's snapshot, breaking unrelated tests. Regenerates the six affected snapshots. - Skip Moxfield URL tests when MOXFIELD_SECRET is not configured and update_database tests when client_secrets.json is absent or invalid, so fork PRs (which receive no repo secrets) and credential-less local runs pass instead of erroring. Co-Authored-By: Claude Fable 5 --- .../tests/__snapshots__/test_views.ambr | 12 ++++++------ MPCAutofill/cardpicker/tests/conftest.py | 15 +++++++++++++++ .../cardpicker/tests/test_integrations.py | 2 ++ MPCAutofill/cardpicker/tests/test_sources.py | 16 ++++++++++++++++ MPCAutofill/pytest.ini | 3 +++ 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr index 4f2b04a30..e94342b1a 100644 --- a/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr +++ b/MPCAutofill/cardpicker/tests/__snapshots__/test_views.ambr @@ -193,7 +193,7 @@ 'CARD': dict({ 'Brainstorm': dict({ 'canonicalArtist': dict({ - 'name': 'Artist 102', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -416,7 +416,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 107', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -702,7 +702,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 109', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -2461,7 +2461,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 67', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -2735,7 +2735,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 66', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, @@ -3087,7 +3087,7 @@ 'cards': list([ dict({ 'canonicalArtist': dict({ - 'name': 'Artist 68', + 'name': 'Artist 0', }), 'canonicalCard': dict({ 'artist': None, diff --git a/MPCAutofill/cardpicker/tests/conftest.py b/MPCAutofill/cardpicker/tests/conftest.py index f401eb86d..ac495bf89 100644 --- a/MPCAutofill/cardpicker/tests/conftest.py +++ b/MPCAutofill/cardpicker/tests/conftest.py @@ -1,7 +1,9 @@ import datetime as dt +import inspect import uuid from typing import Type +import factory as factory_boy import pytest from pytest_elasticsearch import factories from testcontainers.elasticsearch import ElasticSearchContainer @@ -12,6 +14,7 @@ from cardpicker.integrations.game.base import GameIntegration from cardpicker.models import Card, CardTypes, DFCPair, Source, Tag +from cardpicker.tests import factories as factories_module from cardpicker.tests.constants import Cards, DummyIntegration, Sources from cardpicker.tests.factories import ( CanonicalCardFactory, @@ -26,6 +29,18 @@ ELASTICSEARCH_PORT = 9300 # this is the default expected by `elasticsearch_nooproc` +@pytest.fixture(autouse=True) +def reset_factory_sequences() -> None: + """ + Snapshots embed sequence-generated values (e.g. "Artist 0"), so sequences must restart for every test - + otherwise adding or running a subset of tests shifts the numbering and breaks unrelated snapshots. + """ + + for _, factory_class in inspect.getmembers(factories_module, inspect.isclass): + if issubclass(factory_class, factory_boy.django.DjangoModelFactory): + factory_class.reset_sequence(0) + + @pytest.fixture(scope="session") def postgres_container(): postgres = PostgresContainer("postgres:16.0-alpine").with_bind_ports(5432, POSTGRES_PORT) diff --git a/MPCAutofill/cardpicker/tests/test_integrations.py b/MPCAutofill/cardpicker/tests/test_integrations.py index f206ae780..2287b8825 100644 --- a/MPCAutofill/cardpicker/tests/test_integrations.py +++ b/MPCAutofill/cardpicker/tests/test_integrations.py @@ -107,6 +107,8 @@ def test_get_meld_pairs(self): @pytest.mark.parametrize("url", [item.value for item in Decks], ids=[item.name.lower() for item in Decks]) def test_valid_url(self, client, django_settings, snapshot, url: str): + if "moxfield" in url and not conf_settings.MOXFIELD_SECRET: + pytest.skip("MOXFIELD_SECRET is not configured (e.g. fork PRs and environments without credentials)") decklist = MTGIntegration.query_import_site(url) assert decklist assert Counter(decklist.splitlines()) == snapshot diff --git a/MPCAutofill/cardpicker/tests/test_sources.py b/MPCAutofill/cardpicker/tests/test_sources.py index a686c61c6..948490a8c 100644 --- a/MPCAutofill/cardpicker/tests/test_sources.py +++ b/MPCAutofill/cardpicker/tests/test_sources.py @@ -1,4 +1,6 @@ import datetime as dt +import json +from pathlib import Path import freezegun import pytest @@ -21,6 +23,16 @@ DEFAULT_DATE = dt.datetime(2023, 1, 1) +def google_drive_credentials_available() -> bool: + # mirrors the path resolution in `cardpicker.sources.api`. fork PRs and local runs without + # credentials get a missing or unparseable client_secrets.json, so those tests must skip. + try: + with open(Path(__file__).parent.parent.parent / "client_secrets.json") as f: + return bool(json.load(f)) + except (OSError, json.JSONDecodeError): + return False + + class TestAPI: # region constants @@ -354,6 +366,10 @@ def test_unpack_name( # endregion +@pytest.mark.skipif( + not google_drive_credentials_available(), + reason="Google Drive credentials (client_secrets.json) are not available", +) class TestUpdateDatabase: # region tests diff --git a/MPCAutofill/pytest.ini b/MPCAutofill/pytest.ini index 9f62bfe06..b86f91aef 100644 --- a/MPCAutofill/pytest.ini +++ b/MPCAutofill/pytest.ini @@ -1,2 +1,5 @@ [pytest] DJANGO_SETTINGS_MODULE = MPCAutofill.settings +# credential-dependent tests (update_database, Moxfield) skip when secrets are absent - e.g. on fork +# PRs - which leaves their committed snapshots unused; warn rather than fail the run in that case +addopts = --snapshot-warn-unused