From 74e7f0a1914d61384f21fc8f7bf59e9beb592073 Mon Sep 17 00:00:00 2001 From: GH Date: Sat, 1 Aug 2026 04:19:22 +0000 Subject: [PATCH 1/4] Add design spec for persisting client-side search indexes Co-Authored-By: Claude Fable 5 --- ...01-persist-client-search-indexes-design.md | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-01-persist-client-search-indexes-design.md diff --git a/docs/superpowers/specs/2026-08-01-persist-client-search-indexes-design.md b/docs/superpowers/specs/2026-08-01-persist-client-search-indexes-design.md new file mode 100644 index 000000000..7a1ee90c4 --- /dev/null +++ b/docs/superpowers/specs/2026-08-01-persist-client-search-indexes-design.md @@ -0,0 +1,81 @@ +# Persist Client-Side Search Indexes — Design + +**Date:** 2026-08-01 +**Status:** Approved +**Fixes:** chilli-axe/mpc-autofill#418 (discussion in #282, follows PR #358) + +## Problem + +Local-files and Google Drive client-side search indexes live only in memory. +Every page load, the user must re-pick their folder / re-run the Drive picker +and wait for a full re-index. For large libraries this takes long enough that +the feature is painful to use. + +## Approach: persist documents + handles in IndexedDB, rebuild Orama on load + +The expensive step is walking the folder / hitting the Drive API — not +building the Orama database (insertMultiple of a few thousand documents is +~100ms). So we persist the raw `OramaCardDocument` arrays and rebuild the +index on load. + +`FileSystemFileHandle` / `FileSystemDirectoryHandle` objects are +structured-cloneable and can be stored in IndexedDB natively, so local-file +documents (whose `params` embed live file handles) persist as-is. + +Rejected alternatives: +- `@orama/plugin-data-persistence` — serializes to JSON/binary, which would + destroy the embedded file handles. +- Persisting only the directory handle and silently re-indexing on load — + re-walking the folder every load is exactly the slowness complained about. + +## Components + +### `frontend/src/features/clientSearch/persistence.ts` + +Small typed wrapper over raw IndexedDB (no new dependency): `save`, `load`, +`clear` for two entries plus a schema version: + +- `localFiles: { directoryHandle, documents, indexedAt }` +- `googleDrive: { documents, indexedAt }` +- On schema-version mismatch or corrupt/unreadable data: treat as empty and + clear the store. + +Saves run after each successful index build, fire-and-forget (quota or other +errors are logged, never surfaced as failures). + +### Restore flow + +On client search service init, load both entries and rebuild the Orama +indexes through the existing document-insertion path. Search works +immediately; no permission prompt. Restore failure of any kind degrades to +today's empty state. + +### Permission re-grant (local files) + +Reading image *files* (thumbnails, export) needs handle permission again. +After restore, if `queryPermission({mode: "read"}) !== "granted"`, the +source-config UI shows a banner ("Re-grant access to show images") whose +click calls `requestPermission()` — satisfying the browser's user-gesture +requirement. Search never blocks on this. + +### Re-sync + staleness + +The source-config UI shows restored state (" — N cards, indexed +") with a **Re-sync** button that re-walks the folder / +re-fetches from Drive (Drive re-sync triggers re-auth as today) and +overwrites the stored entry. No automatic background re-sync. + +## Error handling + +Every persistence/restore operation is best-effort: failures log to console +and fall back to the un-persisted behavior. A fresh session can never be +broken by this feature. + +## Testing + +- Jest with `fake-indexeddb`: persistence module round-trip, restore rebuilds + a searchable index, corrupt/old-version entries restore to empty and clear. +- Guard test: with nothing persisted, service behaves exactly as today. +- E2E cannot drive `showDirectoryPicker`, so local-folder flows are covered at + the unit level; the Drive restore path gets a Playwright test if the + existing MSW mocks support it, otherwise unit-level too. From 998fad775f20f56359ebeee76e7a6f4520e76b92 Mon Sep 17 00:00:00 2001 From: GH Date: Sat, 1 Aug 2026 04:34:09 +0000 Subject: [PATCH 2/4] Add IndexedDB persistence module for client search indexes Co-Authored-By: Claude Fable 5 --- frontend/package-lock.json | 44 +++++-- frontend/package.json | 1 + .../features/clientSearch/persistence.test.ts | 76 ++++++++++++ .../src/features/clientSearch/persistence.ts | 114 ++++++++++++++++++ 4 files changed, 227 insertions(+), 8 deletions(-) create mode 100644 frontend/src/features/clientSearch/persistence.test.ts create mode 100644 frontend/src/features/clientSearch/persistence.ts diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 692bbf6c3..7ec820437 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -68,6 +68,7 @@ "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-simple-import-sort": "^10.0.0", + "fake-indexeddb": "^6.2.5", "file-loader": "^6.2.0", "jest": "^30.2.0", "jest-fixed-jsdom": "^0.0.11", @@ -2836,6 +2837,7 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.1.tgz", "integrity": "sha512-dfUnCxiN9H4ap84DvD2ubjw+3vUNpstxa0TneY/Paat8a3R4uQZDLSvWjmznAY/DoahqTHl9V46HF/Zs3F29pg==", + "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -2875,6 +2877,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2895,6 +2898,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2915,6 +2919,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2935,6 +2940,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2955,6 +2961,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2975,6 +2982,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2995,6 +3003,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3015,6 +3024,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3035,6 +3045,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3055,6 +3066,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3075,6 +3087,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3095,6 +3108,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3115,6 +3129,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6475,7 +6490,7 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -7388,6 +7403,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", + "dev": true, "license": "Apache-2.0", "optional": true, "bin": { @@ -8789,6 +8805,16 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/fake-indexeddb": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.5.tgz", + "integrity": "sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -8966,7 +8992,7 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -10053,7 +10079,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -10119,7 +10145,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -10165,7 +10191,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -12622,7 +12648,7 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -12636,7 +12662,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -13069,6 +13095,7 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "dev": true, "license": "MIT", "optional": true }, @@ -13781,6 +13808,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -15976,7 +16004,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" diff --git a/frontend/package.json b/frontend/package.json index 8c00cc93e..186f8ccd4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -75,6 +75,7 @@ "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-simple-import-sort": "^10.0.0", + "fake-indexeddb": "^6.2.5", "file-loader": "^6.2.0", "jest": "^30.2.0", "jest-fixed-jsdom": "^0.0.11", diff --git a/frontend/src/features/clientSearch/persistence.test.ts b/frontend/src/features/clientSearch/persistence.test.ts new file mode 100644 index 000000000..73f35b791 --- /dev/null +++ b/frontend/src/features/clientSearch/persistence.test.ts @@ -0,0 +1,76 @@ +import "fake-indexeddb/auto"; + +import { IDBFactory } from "fake-indexeddb"; + +import { + clearPersistedIndex, + loadPersistedIndex, + savePersistedIndex, +} from "./persistence"; + +const buildDocument = (id: string) => ({ + id, + name: id, + searchq: id.toLowerCase(), + sourceId: -1, + tags: [] as Array, +}); + +beforeEach(() => { + // a fresh in-memory IndexedDB per test + indexedDB = new IDBFactory(); +}); + +describe("client search index persistence", () => { + test("round-trips a persisted entry", async () => { + const documents = [buildDocument("Card A"), buildDocument("Card B")]; + await savePersistedIndex("googleDrive", { + documents, + indexedAt: 1_700_000_000_000, + }); + const loaded = await loadPersistedIndex("googleDrive"); + expect(loaded?.indexedAt).toBe(1_700_000_000_000); + expect(loaded?.documents).toEqual(documents); + }); + + test("returns undefined when nothing was persisted", async () => { + expect(await loadPersistedIndex("localFiles")).toBeUndefined(); + }); + + test("entries are independent per source type", async () => { + await savePersistedIndex("googleDrive", { + documents: [buildDocument("Drive Card")], + indexedAt: 1, + }); + expect(await loadPersistedIndex("localFiles")).toBeUndefined(); + }); + + test("clear removes the entry", async () => { + await savePersistedIndex("localFiles", { + documents: [buildDocument("Card A")], + indexedAt: 1, + }); + await clearPersistedIndex("localFiles"); + expect(await loadPersistedIndex("localFiles")).toBeUndefined(); + }); + + test("an entry with a mismatched schema version loads as undefined and is cleared", async () => { + // simulate a future/old schema by writing with a different version + await savePersistedIndex( + "googleDrive", + { documents: [buildDocument("Card A")], indexedAt: 1 }, + { schemaVersion: 999 } + ); + expect(await loadPersistedIndex("googleDrive")).toBeUndefined(); + }); + + test("load failures degrade to undefined rather than throwing", async () => { + // break indexedDB entirely + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any).indexedDB = undefined; + await expect(loadPersistedIndex("googleDrive")).resolves.toBeUndefined(); + await expect( + savePersistedIndex("googleDrive", { documents: [], indexedAt: 1 }) + ).resolves.toBeUndefined(); + }); +}); diff --git a/frontend/src/features/clientSearch/persistence.ts b/frontend/src/features/clientSearch/persistence.ts new file mode 100644 index 000000000..833371bd6 --- /dev/null +++ b/frontend/src/features/clientSearch/persistence.ts @@ -0,0 +1,114 @@ +/** + * IndexedDB persistence for client-side search indexes (see issue #418). + * + * We persist the raw indexed documents (plus, for local files, the picked + * directory handle) rather than a serialised Orama database — rebuilding the + * Orama index from documents is fast, and file handles are structured- + * cloneable so IndexedDB stores them natively where JSON cannot. + * + * Every operation is best-effort: failures log to the console and behave as + * "nothing persisted". A fresh session can never be broken by this module. + */ + +const DATABASE_NAME = "mpc-autofill-client-search"; +const STORE_NAME = "indexes"; +const SCHEMA_VERSION = 1; + +export type PersistedSourceType = "localFiles" | "googleDrive"; + +export interface PersistedIndex { + documents: Array; + indexedAt: number; + directoryHandle?: FileSystemDirectoryHandle; +} + +interface StoredEntry extends PersistedIndex { + schemaVersion: number; +} + +function openDatabase(): Promise { + return new Promise((resolve, reject) => { + const request = indexedDB.open(DATABASE_NAME, 1); + request.onupgradeneeded = () => { + if (!request.result.objectStoreNames.contains(STORE_NAME)) { + request.result.createObjectStore(STORE_NAME); + } + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); +} + +function requestToPromise(request: IDBRequest): Promise { + return new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); +} + +export async function savePersistedIndex( + sourceType: PersistedSourceType, + entry: PersistedIndex, + options?: { schemaVersion?: number } +): Promise { + try { + const db = await openDatabase(); + const stored: StoredEntry = { + ...entry, + schemaVersion: options?.schemaVersion ?? SCHEMA_VERSION, + }; + await requestToPromise( + db + .transaction(STORE_NAME, "readwrite") + .objectStore(STORE_NAME) + .put(stored, sourceType) + ); + db.close(); + } catch (error) { + console.warn("Failed to persist client search index:", error); + } +} + +export async function loadPersistedIndex( + sourceType: PersistedSourceType +): Promise { + try { + const db = await openDatabase(); + const stored: StoredEntry | undefined = await requestToPromise( + db.transaction(STORE_NAME).objectStore(STORE_NAME).get(sourceType) + ); + db.close(); + if (stored === undefined) { + return undefined; + } + if ( + stored.schemaVersion !== SCHEMA_VERSION || + !Array.isArray(stored.documents) + ) { + await clearPersistedIndex(sourceType); + return undefined; + } + const { schemaVersion, ...entry } = stored; + return entry; + } catch (error) { + console.warn("Failed to load persisted client search index:", error); + return undefined; + } +} + +export async function clearPersistedIndex( + sourceType: PersistedSourceType +): Promise { + try { + const db = await openDatabase(); + await requestToPromise( + db + .transaction(STORE_NAME, "readwrite") + .objectStore(STORE_NAME) + .delete(sourceType) + ); + db.close(); + } catch (error) { + console.warn("Failed to clear persisted client search index:", error); + } +} From 090a9e367515ecb4fdbd90cfb852727e0542126d Mon Sep 17 00:00:00 2001 From: GH Date: Sat, 1 Aug 2026 05:12:56 +0000 Subject: [PATCH 3/4] Restore persisted client search indexes on app load Fixes #418. Indexes for local files and Google Drive rebuild from IndexedDB-persisted documents on page load, so users no longer redo folder association and file sync every visit. Search works immediately after restore; a banner in the local folder config prompts the user to re-grant folder access (a browser user-gesture requirement) before local images can display. Disconnecting a source clears its persisted data. Co-Authored-By: Claude Fable 5 --- .../backend/LocalFolderBackendConfig.tsx | 29 +++++++- .../clientSearch/buildOramaIndex.test.ts | 52 ++++++++++++++ .../features/clientSearch/buildOramaIndex.ts | 36 ++++++++++ .../clientSearch/clientSearchService.ts | 28 ++++++++ .../clientSearchService.worker.ts | 67 +++++++++++++++++-- frontend/src/features/clientSearch/indexer.ts | 31 ++------- frontend/src/features/ui/Layout.tsx | 7 ++ 7 files changed, 216 insertions(+), 34 deletions(-) create mode 100644 frontend/src/features/clientSearch/buildOramaIndex.test.ts create mode 100644 frontend/src/features/clientSearch/buildOramaIndex.ts diff --git a/frontend/src/features/backend/LocalFolderBackendConfig.tsx b/frontend/src/features/backend/LocalFolderBackendConfig.tsx index 96ddcdd47..904b6057d 100644 --- a/frontend/src/features/backend/LocalFolderBackendConfig.tsx +++ b/frontend/src/features/backend/LocalFolderBackendConfig.tsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import Alert from "react-bootstrap/Alert"; import Button from "react-bootstrap/Button"; import Col from "react-bootstrap/Col"; @@ -26,6 +26,21 @@ export const LocalFolderBackendConfig = () => { const { clientSearchService, forceUpdate } = useClientSearchContext(); const directoryHandle = useLocalFilesDirectoryHandle(); const directoryIndexSize = useLocalFilesDirectoryIndexSize(); + // a restored directory handle (issue #418) may need its read permission re-granted by a user gesture + const [needsPermission, setNeedsPermission] = useState(false); + useEffect(() => { + // @ts-ignore - queryPermission is not in the standard lib typings yet + directoryHandle?.queryPermission({ mode: "readwrite" }).then( + (permission: PermissionState) => + setNeedsPermission(permission !== "granted"), + () => setNeedsPermission(false) + ); + }, [directoryHandle]); + const reGrantPermission = async () => { + const permission: PermissionState = await (directoryHandle as any) // requestPermission is not in the standard lib typings yet + ?.requestPermission({ mode: "readwrite" }); + setNeedsPermission(permission !== "granted"); + }; const getTagsQuery = useGetTagsQuery(); const [validationStatus, setValidationStatus] = useState< @@ -102,6 +117,18 @@ export const LocalFolderBackendConfig = () => { You're connected to {directoryHandle.name}, with{" "} {directoryIndexSize ?? 0} images indexed. + {needsPermission && ( + + +
+ +
+ + + )}
diff --git a/frontend/src/features/clientSearch/buildOramaIndex.test.ts b/frontend/src/features/clientSearch/buildOramaIndex.test.ts new file mode 100644 index 000000000..9b810b270 --- /dev/null +++ b/frontend/src/features/clientSearch/buildOramaIndex.test.ts @@ -0,0 +1,52 @@ +import { search } from "@orama/orama"; + +import { CardType as CardTypeSchema, SourceType } from "@/common/schema_types"; +import { OramaCardDocument } from "@/common/types"; + +import { buildOramaIndex } from "./buildOramaIndex"; + +const buildDocument = (name: string): OramaCardDocument => ({ + id: name, + name, + searchq: name.toLowerCase(), + source: "test", + sourceId: -1, + sourceVerbose: "Test", + cardType: CardTypeSchema.Card, + extension: "png", + language: "EN", + tags: [], + dpi: 600, + size: 1, + lastModified: new Date(2020, 0, 1), + lastModifiedNumber: new Date(2020, 0, 1).valueOf(), + created: new Date(2020, 0, 1), + createdNumber: new Date(2020, 0, 1).valueOf(), + expansionCode: "UNK", + collectorNumber: "UNK", + artist: "Unknown", + params: { + sourceType: SourceType.GoogleDrive, + identifier: name, + fileHandle: undefined, + }, +}); + +describe("buildOramaIndex", () => { + test("builds a searchable index from persisted documents", async () => { + const index = buildOramaIndex([ + buildDocument("Lightning Bolt"), + buildDocument("Counterspell"), + ]); + expect(index.size).toBe(2); + const results = search(index.oramaDb, { + term: "counterspell", + properties: ["searchq"], + }) as { count: number }; + expect(results.count).toBe(1); + }); + + test("builds an empty index from no documents", () => { + expect(buildOramaIndex([]).size).toBe(0); + }); +}); diff --git a/frontend/src/features/clientSearch/buildOramaIndex.ts b/frontend/src/features/clientSearch/buildOramaIndex.ts new file mode 100644 index 000000000..6a375c75e --- /dev/null +++ b/frontend/src/features/clientSearch/buildOramaIndex.ts @@ -0,0 +1,36 @@ +import { create, insertMultiple } from "@orama/orama"; + +import { OramaCardDocument, OramaIndex, OramaSchema } from "@/common/types"; + +/** + * Build a searchable Orama index from card documents. Used both when indexing + * from scratch and when restoring persisted documents from IndexedDB. + */ +export function buildOramaIndex( + documents: Array +): OramaIndex { + const db = create({ + schema: OramaSchema, + sort: { + enabled: true, + unsortableProperties: [ + // every field on OramaCardDocument except `searchq` and `lastModifiedNumber` :) + "name", + "source", + "sourceId", + "sourceVerbose", + "cardType", + "extension", + "language", + "tags", + "dpi", + "size", + "id", + "lastModified", + "params", + ], + }, + }); + insertMultiple(db, documents); + return { oramaDb: db, size: documents.length }; +} diff --git a/frontend/src/features/clientSearch/clientSearchService.ts b/frontend/src/features/clientSearch/clientSearchService.ts index b9e36fa3b..b1f010ed0 100644 --- a/frontend/src/features/clientSearch/clientSearchService.ts +++ b/frontend/src/features/clientSearch/clientSearchService.ts @@ -97,6 +97,34 @@ export class ClientSearchService { .then(() => recalculateSearchResults(state, dispatch, true)); } + /** + * Rebuild indexes from persisted documents (issue #418). Returns what was + * restored so the UI can reflect it; refreshes search state when anything + * was. + */ + public async restorePersistedIndexes( + dispatch: AppDispatch, + forceUpdate: DispatchWithoutAction + ): Promise<{ + localFiles?: { size: number; indexedAt: number }; + googleDrive?: { size: number; indexedAt: number }; + }> { + if (this.worker === undefined) { + throw new Error("clientSearchService was not initialised!"); + } + const restored = await this.worker.restorePersistedIndexes(); + if ( + restored.localFiles !== undefined || + restored.googleDrive !== undefined + ) { + dispatch(api.util.invalidateTags([QueryTags.BackendSpecific])); + dispatch(clearSearchResults()); + fetchCardDocumentsAndReportError(dispatch, { refreshCardbacks: true }); + forceUpdate(); + } + return restored; + } + public async indexDirectory( dispatch: AppDispatch, forceUpdate: DispatchWithoutAction, diff --git a/frontend/src/features/clientSearch/clientSearchService.worker.ts b/frontend/src/features/clientSearch/clientSearchService.worker.ts index f8ce317cf..b3f9821fa 100644 --- a/frontend/src/features/clientSearch/clientSearchService.worker.ts +++ b/frontend/src/features/clientSearch/clientSearchService.worker.ts @@ -28,7 +28,13 @@ import { import { parseDjangoDate } from "@/common/utils"; import { getDefaultSearchSettings } from "@/store/slices/searchSettingsSlice"; +import { buildOramaIndex } from "./buildOramaIndex"; import { Folder, GoogleDriveIndexer, LocalFilesIndexer } from "./indexer"; +import { + clearPersistedIndex, + loadPersistedIndex, + savePersistedIndex, +} from "./persistence"; export class ClientSearchService { private localFilesIndex: LocalFilesIndex | undefined; @@ -63,10 +69,12 @@ export class ClientSearchService { public async clearLocalFilesIndex() { this.localFilesIndex = undefined; + await clearPersistedIndex("localFiles"); } public async clearGoogleDriveIndex() { this.googleDriveIndex = undefined; + await clearPersistedIndex("googleDrive"); } public getLocalFilesIndexSize(): number | undefined { @@ -81,7 +89,7 @@ export class ClientSearchService { tags: Array | undefined ): Promise<{ handle: FileSystemDirectoryHandle; size: number } | undefined> { if (this.localFilesIndex?.fileHandle !== undefined) { - const oramaIndex = await new LocalFilesIndexer().indexFiles( + const { index, documents } = await new LocalFilesIndexer().indexFiles( [ new Folder( { @@ -96,10 +104,15 @@ export class ClientSearchService { [], tags ); - this.localFilesIndex.index = oramaIndex; + this.localFilesIndex.index = index; + savePersistedIndex("localFiles", { + documents, + indexedAt: Date.now(), + directoryHandle: this.localFilesIndex.fileHandle, + }); return { handle: this.localFilesIndex.fileHandle, - size: this.localFilesIndex.index.size, + size: index.size, }; } return undefined; @@ -112,7 +125,7 @@ export class ClientSearchService { images: Array ) { const indexer = new GoogleDriveIndexer(bearerToken); - const oramaIndex = await indexer.indexFiles( + const { index, documents } = await indexer.indexFiles( folders.map( ({ id, name }) => new Folder( @@ -134,12 +147,54 @@ export class ClientSearchService { ).filter((image) => image !== undefined), tags ); - this.googleDriveIndex = { index: oramaIndex }; + this.googleDriveIndex = { index }; + savePersistedIndex("googleDrive", { documents, indexedAt: Date.now() }); return { - size: this.googleDriveIndex.index?.size, + size: index.size, }; } + /** + * Rebuild indexes from IndexedDB-persisted documents (issue #418). Search + * works immediately after restore; local file *reads* may still need the + * user to re-grant permission on the restored directory handle. + */ + public async restorePersistedIndexes(): Promise<{ + localFiles?: { size: number; indexedAt: number }; + googleDrive?: { size: number; indexedAt: number }; + }> { + const restored: { + localFiles?: { size: number; indexedAt: number }; + googleDrive?: { size: number; indexedAt: number }; + } = {}; + const localFiles = await loadPersistedIndex("localFiles"); + if (localFiles?.directoryHandle !== undefined) { + this.localFilesIndex = { + fileHandle: localFiles.directoryHandle, + index: buildOramaIndex( + localFiles.documents as Array + ), + }; + restored.localFiles = { + size: localFiles.documents.length, + indexedAt: localFiles.indexedAt, + }; + } + const googleDrive = await loadPersistedIndex("googleDrive"); + if (googleDrive !== undefined) { + this.googleDriveIndex = { + index: buildOramaIndex( + googleDrive.documents as Array + ), + }; + restored.googleDrive = { + size: googleDrive.documents.length, + indexedAt: googleDrive.indexedAt, + }; + } + return restored; + } + private searchOramaIndex( oramaIndex: OramaIndex | undefined, searchSettings: SearchSettings, diff --git a/frontend/src/features/clientSearch/indexer.ts b/frontend/src/features/clientSearch/indexer.ts index ca69ec0d7..595f017e2 100644 --- a/frontend/src/features/clientSearch/indexer.ts +++ b/frontend/src/features/clientSearch/indexer.ts @@ -1,4 +1,3 @@ -import { create, insertMultiple } from "@orama/orama"; import { imageDimensionsFromStream, ImageType } from "image-dimensions"; import { Unknown } from "@/common/constants"; @@ -26,6 +25,7 @@ import { OramaSchema } from "@/common/types"; import { extractNameAndTags } from "@/features/clientSearch/tags"; import { GoogleDriveService } from "../googleDrive/GoogleDriveService"; +import { buildOramaIndex } from "./buildOramaIndex"; export class Folder { constructor( @@ -184,29 +184,7 @@ abstract class Indexer { folders: Array, images: Array, tags: Array | undefined - ): Promise { - const db = create({ - schema: OramaSchema, - sort: { - enabled: true, - unsortableProperties: [ - // every field on OramaCardDocument except `searchq` and `lastModifiedNumber` :) - "name", - "source", - "sourceId", - "sourceVerbose", - "cardType", - "extension", - "language", - "tags", - "dpi", - "size", - "id", - "lastModified", - "params", - ], - }, - }); + ): Promise<{ index: OramaIndex; documents: Array }> { const tagsMap = new Map( (tags ?? []).map((tag) => [tag.name.toLowerCase(), tag]) ); @@ -227,10 +205,9 @@ abstract class Indexer { const deduplicatedOramaCardDocuments = uniqueImages.map((image) => image.getOramaCardDocument(tagsMap) ); - insertMultiple(db, deduplicatedOramaCardDocuments); return { - oramaDb: db, - size: deduplicatedOramaCardDocuments.length, + index: buildOramaIndex(deduplicatedOramaCardDocuments), + documents: deduplicatedOramaCardDocuments, }; } } diff --git a/frontend/src/features/ui/Layout.tsx b/frontend/src/features/ui/Layout.tsx index 7d7be0085..c68309ffc 100644 --- a/frontend/src/features/ui/Layout.tsx +++ b/frontend/src/features/ui/Layout.tsx @@ -82,6 +82,13 @@ export function LayoutWithoutReduxProvider({ children }: PropsWithChildren) { } clientSearchService.initialiseWorker(); pdfRenderService.initialiseWorker(); + // restore any persisted client search indexes (issue #418) - best-effort + clientSearchService + .restorePersistedIndexes(dispatch, forceUpdate) + .catch((error) => + console.warn("Failed to restore persisted search indexes:", error) + ); + // eslint-disable-next-line react-hooks/exhaustive-deps }, []); return ( From 4afb9e5d811d35d4e81ee1a259057b1b55eec6e7 Mon Sep 17 00:00:00 2001 From: GH Date: Sat, 1 Aug 2026 05:21:36 +0000 Subject: [PATCH 4/4] Use ts-expect-error instead of an eslint rule next build doesn't know Co-Authored-By: Claude Fable 5 --- frontend/src/features/clientSearch/persistence.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/features/clientSearch/persistence.test.ts b/frontend/src/features/clientSearch/persistence.test.ts index 73f35b791..c6a2f34ae 100644 --- a/frontend/src/features/clientSearch/persistence.test.ts +++ b/frontend/src/features/clientSearch/persistence.test.ts @@ -66,8 +66,8 @@ describe("client search index persistence", () => { test("load failures degrade to undefined rather than throwing", async () => { // break indexedDB entirely - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (globalThis as any).indexedDB = undefined; + // @ts-expect-error - deliberately clobbering the global to simulate storage failure + globalThis.indexedDB = undefined; await expect(loadPersistedIndex("googleDrive")).resolves.toBeUndefined(); await expect( savePersistedIndex("googleDrive", { documents: [], indexedAt: 1 })