feat(schema): add v3 cache invalidation

docs/schema-upgrade.md

@@ -43,9 +43,18 @@ across query/list/search/evidence business paths.
 
 The v1 -> v2 archive upgrader removes embedded archive `index.db` and legacy
 `fts.db` as derived search index data and preserves important archive content
-and the mutation token. It must refuse active coordinator state for the target
-archive and non-search-index overlays, because those can represent uncommitted
-important data.
+and the mutation token.
+
+The v2 -> v3 archive upgrader separates index artifacts from index caches. It
+keeps important archive data, rebuilds chapter FTS index artifacts from the
+archive source/summary/object data already present in `database.db`, drops the
+old `archive_index_settings` table, and deletes embedded `index.db` / legacy
+`fts.db` caches. It does not create embedding artifacts, because older schemas
+never stored them as important data.
+
+Archive upgraders must refuse active coordinator state for the target archive
+and non-search-index overlays, because those can represent uncommitted important
+data.
 
 ## Home Gate
 
@@ -78,18 +87,25 @@ code and tests when adding new home SQLite files:
   - archive coordinator external search index cache workspace referenced by
     `entry_overlays(entry_path = 'index.db')`.
 
-For v1 -> v2, derived home data is deleted or invalidated: query/search caches,
-external cache, GC state, build queue SQLite/cache when safe, library aggregate
-indexes, external archive search index overlays/workspaces for `index.db` or
-legacy `fts.db`, and
-orphaned SQLite materialization cache overlays whose archive file no longer
-exists. The upgrader must block when active GC, build job, worker lease,
+For home schema upgrades, derived home data is deleted or invalidated:
+query/search caches, external cache, GC state, build queue SQLite/cache when
+safe, library aggregate indexes, external archive search index
+overlays/workspaces for `index.db` or legacy `fts.db`, and orphaned SQLite
+materialization cache overlays whose archive file no longer exists. The v2 -> v3
+home upgrader uses the same cleanup boundary because the index-cache semantics
+changed. The upgrader must block when active GC, build job, worker lease,
 coordinator owner/lock/sqlite lease/commit lock, or remaining non-derived
 overlay state is present.
 
 Pure information commands such as `wg --version` and help rendering must not open
 home SQLite and must not trigger schema upgrade.
 
+Search index caches also carry their own `search_index_state.version`. Opening a
+cache whose version is missing, unreadable, or different from the current search
+index version must delete that cache and treat it as missing. Write paths may
+then recreate the cache from artifacts; read paths must report the missing-cache
+state instead of querying stale SQLite.
+
 After a home `core.sqlite` file is confirmed current, the home gate may memoize
 that result inside the current process for hot gated access. The memo must be
 bound to both the resolved `core.sqlite` path and a file fingerprint (`dev`,

packages/cli/src/commands/archive-command/index.test.ts

@@ -1,10 +1,15 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { mkdtemp, rm } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 import {
   findWikiGraphLibraryObjects,
   listWikiGraphLibraryObjects,
 } from "wiki-graph-core";
 import type * as WikiGraphCore from "wiki-graph-core";
+import { setWikiGraphStateDirectoryPathForTesting } from "../../../../core/src/runtime/common/wiki-graph/dir.js";
 import { parseCLIArguments } from "../../args/index.js";
 import { writeFindHits } from "../archive-output/index.js";
 import { runArchiveCommand } from "./index.js";
@@ -66,7 +71,11 @@ vi.mock("./run/index.js", async (importOriginal) => {
 });
 vi.mock("./run/scope.js", () => ({ resolveArchiveChapterScope: vi.fn() }));
 
-beforeEach(() => {
+let testStateDir: string | undefined;
+
+beforeEach(async () => {
+  testStateDir = await mkdtemp(join(tmpdir(), "wikigraph-archive-command-"));
+  setWikiGraphStateDirectoryPathForTesting(testStateDir);
   vi.clearAllMocks();
   vi.mocked(findWikiGraphLibraryObjects).mockResolvedValue({
     chapters: null,
@@ -92,6 +101,14 @@ beforeEach(() => {
   });
 });
 
+afterEach(async () => {
+  setWikiGraphStateDirectoryPathForTesting(undefined);
+  if (testStateDir !== undefined) {
+    await rm(testStateDir, { force: true, recursive: true });
+    testStateDir = undefined;
+  }
+});
+
 describe("runArchiveCommand library nested scopes", () => {
   it("searches library-wide triple pattern scopes through the library index", async () => {
     const parsed = parseCLIArguments([

packages/core/src/document/directory/search-index.test.ts

@@ -0,0 +1,128 @@
+import { mkdtemp, rm, stat } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+
+import { describe, expect, it } from "vitest";
+
+import { DirectoryDocument } from "./index.js";
+import { SEARCH_INDEX_VERSION } from "../../retrieval/search-index/search/types.js";
+
+async function withDocument(
+  operation: (document: DirectoryDocument, path: string) => Promise<void>,
+): Promise<void> {
+  const path = await mkdtemp(join(tmpdir(), "wikigraph-search-index-"));
+
+  try {
+    const document = await DirectoryDocument.open(path);
+
+    try {
+      await operation(document, path);
+    } finally {
+      await document.release();
+    }
+  } finally {
+    await rm(path, { force: true, recursive: true });
+  }
+}
+
+describe("DirectoryDocument search index cache", () => {
+  it("deletes incompatible cache on read", async () => {
+    await withDocument(async (document, path) => {
+      await writeIncompatibleCache(document);
+
+      await expect(
+        document.readSearchIndexDatabase(() => "unreachable"),
+      ).rejects.toThrow("Search index cache is missing: index.db");
+      await expect(stat(join(path, "index.db"))).rejects.toThrow();
+    });
+  });
+
+  it("reinitializes incompatible cache on write", async () => {
+    await withDocument(async (document) => {
+      await writeIncompatibleCache(document);
+
+      const version = await document.writeSearchIndexDatabase(
+        async (database) =>
+          await database.queryOne(
+            `
+              SELECT value
+              FROM search_index_state
+              WHERE key = 'version'
+            `,
+            undefined,
+            (row) => String(row.value),
+          ),
+      );
+
+      expect(version).toBeUndefined();
+    });
+  });
+
+  it("serializes concurrent incompatible cache reads and writes", async () => {
+    await withDocument(async (document, path) => {
+      await writeIncompatibleCache(document);
+
+      let releaseWriter!: () => void;
+      let resolveWriterEntered!: () => void;
+      const writerEntered = new Promise<void>((resolveEntered) => {
+        resolveWriterEntered = resolveEntered;
+      });
+      const writer = document.writeSearchIndexDatabase(async (database) => {
+        resolveWriterEntered();
+        await new Promise<void>((resolveWriter) => {
+          releaseWriter = resolveWriter;
+        });
+        await database.run(
+          `
+            INSERT OR REPLACE INTO search_index_state(key, value)
+            VALUES ('version', ?), ('concurrent', 'writer')
+          `,
+          [SEARCH_INDEX_VERSION],
+        );
+      });
+      void writer.catch(() => {
+        resolveWriterEntered();
+      });
+
+      try {
+        await writerEntered;
+        const reader = document.readSearchIndexDatabase(async (database) => {
+          const value = await database.queryOne(
+            `
+              SELECT value
+              FROM search_index_state
+              WHERE key = 'concurrent'
+            `,
+            undefined,
+            (row) => String(row.value),
+          );
+
+          return value;
+        });
+
+        releaseWriter();
+
+        await expect(reader).resolves.toBe("writer");
+        await expect(writer).resolves.toBeUndefined();
+        await expect(stat(join(path, "index.db"))).resolves.toBeDefined();
+      } catch (error) {
+        releaseWriter?.();
+        await writer.catch(() => undefined);
+        throw error;
+      }
+    });
+  });
+});
+
+async function writeIncompatibleCache(
+  document: DirectoryDocument,
+): Promise<void> {
+  await document.writeSearchIndexDatabase(async (database) => {
+    await database.run(
+      `
+        INSERT INTO search_index_state(key, value)
+        VALUES ('version', 'old')
+      `,
+    );
+  });
+}

packages/core/src/document/directory/search-index.ts

@@ -1,5 +1,5 @@
 import { stat } from "fs/promises";
-import { join } from "path";
+import { join, resolve } from "path";
 
 import { isNodeError } from "../../utils/node-error.js";
 import { Database } from "../database.js";
@@ -8,21 +8,53 @@ import {
   SEARCH_INDEX_TEXT_SENTENCE_RECORDS_COLUMNS_SQL,
 } from "../schema.js";
 import type { DocumentFileStore } from "./types.js";
+import { SEARCH_INDEX_VERSION } from "../../retrieval/search-index/search/types.js";
+
+const searchIndexLifecycleLocks = new Map<string, Promise<void>>();
 
 export async function openSearchIndexDatabase<T>(input: {
   readonly documentPath: string;
   readonly fileStore: DocumentFileStore;
   readonly operation: (database: Database) => Promise<T> | T;
   readonly readonly: boolean;
 }): Promise<T> {
-  const databasePath =
+  return await withSearchIndexLifecycleLock(resolve(input.documentPath), () =>
+    openSearchIndexDatabaseLocked(input),
+  );
+}
+
+async function openSearchIndexDatabaseLocked<T>(input: {
+  readonly documentPath: string;
+  readonly fileStore: DocumentFileStore;
+  readonly operation: (database: Database) => Promise<T> | T;
+  readonly readonly: boolean;
+}): Promise<T> {
+  let databasePath =
     input.fileStore.resolveSearchIndexDatabasePath === undefined
       ? join(input.documentPath, "index.db")
       : await input.fileStore.resolveSearchIndexDatabasePath(
           input.documentPath,
         );
-  const shouldInitialize =
+  let shouldInitialize =
     !input.readonly && (await isMissingOrEmptyFile(databasePath));
+
+  if (
+    !shouldInitialize &&
+    !(await isSearchIndexDatabaseCompatible(databasePath))
+  ) {
+    await deleteSearchIndexDatabaseFile(input.fileStore, input.documentPath);
+    if (input.readonly) {
+      throw new Error("Search index cache is missing: index.db");
+    }
+    databasePath =
+      input.fileStore.resolveSearchIndexDatabasePath === undefined
+        ? join(input.documentPath, "index.db")
+        : await input.fileStore.resolveSearchIndexDatabasePath(
+            input.documentPath,
+          );
+    shouldInitialize = true;
+  }
+
   const database = await Database.open(
     databasePath,
     shouldInitialize ? SEARCH_INDEX_SCHEMA_SQL : "",
@@ -45,6 +77,30 @@ export async function openSearchIndexDatabase<T>(input: {
   }
 }
 
+async function withSearchIndexLifecycleLock<T>(
+  key: string,
+  operation: () => Promise<T>,
+): Promise<T> {
+  const previous = searchIndexLifecycleLocks.get(key) ?? Promise.resolve();
+  let release!: () => void;
+  const lock = new Promise<void>((resolveLock) => {
+    release = resolveLock;
+  });
+  const current = previous.then(() => lock);
+  searchIndexLifecycleLocks.set(key, current);
+
+  await previous;
+
+  try {
+    return await operation();
+  } finally {
+    release();
+    if (searchIndexLifecycleLocks.get(key) === current) {
+      searchIndexLifecycleLocks.delete(key);
+    }
+  }
+}
+
 async function isMissingOrEmptyFile(path: string): Promise<boolean> {
   const stats = await stat(path).catch((error: unknown) => {
     if (isNodeError(error) && error.code === "ENOENT") {
@@ -57,6 +113,51 @@ async function isMissingOrEmptyFile(path: string): Promise<boolean> {
   return stats === undefined || stats.size === 0;
 }
 
+async function isSearchIndexDatabaseCompatible(
+  databasePath: string,
+): Promise<boolean> {
+  const database = await Database.open(databasePath, "", {
+    readonly: true,
+  }).catch(() => undefined);
+
+  if (database === undefined) {
+    return false;
+  }
+
+  try {
+    const version = await database
+      .queryOne(
+        `
+          SELECT value
+          FROM search_index_state
+          WHERE key = 'version'
+        `,
+        undefined,
+        (row) => String(row.value),
+      )
+      .catch(() => undefined);
+
+    return version === SEARCH_INDEX_VERSION;
+  } finally {
+    await database.close();
+  }
+}
+
+async function deleteSearchIndexDatabaseFile(
+  fileStore: DocumentFileStore,
+  documentPath: string,
+): Promise<void> {
+  try {
+    await fileStore.deleteFile(join(documentPath, "index.db"));
+  } catch (error) {
+    if (isNodeError(error) && error.code === "ENOENT") {
+      return;
+    }
+
+    throw error;
+  }
+}
+
 async function migrateSearchIndexSchema(database: Database): Promise<void> {
   await ensureColumn(
     database,

packages/core/src/document/home-schema-upgrade.ts

@@ -13,7 +13,7 @@ import { isNodeError } from "../utils/node-error.js";
 
 import { Database } from "./database.js";
 
-const CURRENT_HOME_SCHEMA_VERSION = 2;
+const CURRENT_HOME_SCHEMA_VERSION = 3;
 const LOCK_STALE_TIMEOUT_MS = 5 * 60 * 1000;
 const SEARCH_INDEX_DATABASE_PATH = "index.db";
 const LEGACY_SEARCH_INDEX_DATABASE_PATH = "fts.db";

packages/core/src/document/schema.ts

@@ -21,11 +21,6 @@ export const SCHEMA_SQL = `
     value INTEGER NOT NULL
   );
 
-  CREATE TABLE IF NOT EXISTS archive_index_settings (
-    id INTEGER PRIMARY KEY CHECK (id = 1),
-    fts_embedded INTEGER NOT NULL DEFAULT 0
-  );
-
   CREATE TABLE IF NOT EXISTS graph_build_parameters (
     hash TEXT PRIMARY KEY,
     prompt TEXT NOT NULL,