fix: address index artifact review notes

packages/cli/src/args/uri/chapter/routing.ts

@@ -198,7 +198,7 @@ function parseChapterIndexArtifactUriArguments(
       throw new Error(
         withHelpRoute(
           `The chapter index artifact does not support \`${action}\`. Use get, build, or delete.`,
-          "wg <chapter-uri>/index --help",
+          helpRoute,
         ),
       );
   }

packages/cli/src/commands/library.ts

@@ -147,7 +147,7 @@ export async function runLibraryCommand(
       await writer.write({
         json: { type: "started" },
         kind: "lifecycle",
-        text: "library index cache sync started\nsteps: collecting -> clearing -> indexing-text -> indexing-dense -> finalizing",
+        text: "library index cache sync started\nsteps: collecting -> clearing -> indexing-text -> indexing-objects -> indexing-dense -> finalizing",
       });
       const state = await rebuildWikiGraphLibraryIndex(
         args.target,

packages/cli/src/commands/queue/worker.ts

@@ -26,6 +26,7 @@ import { WikiGraphArchiveFile } from "wiki-graph-core";
 import type {
   GuaranteedRequest,
   GuaranteedRequestController,
+  ReadonlyDocument,
 } from "wiki-graph-core";
 import type { LLMessage } from "wiki-graph-core";
 
@@ -73,7 +74,7 @@ async function executeBuildJobWithLogging(
   context: BuildJobExecutionContext,
 ): Promise<void> {
   if (isIndexArtifactBuildTarget(job.target)) {
-    await executeIndexArtifactBuildJob(job, reporter);
+    await executeIndexArtifactBuildJob(job, reporter, context);
     return;
   }
 
@@ -334,6 +335,7 @@ async function executeGenerationBuildJob(
 async function executeIndexArtifactBuildJob(
   job: BuildJob,
   reporter: BuildJobProgressReporter,
+  context: BuildJobExecutionContext,
 ): Promise<void> {
   await reporter.stepStarted(job.target);
   const snapshot = await readIndexArtifactJobSnapshot(job);
@@ -352,6 +354,7 @@ async function executeIndexArtifactBuildJob(
   if (job.target === "index-fts") {
     await new WikiGraphArchiveFile(job.archivePath).write(async (document) => {
       assertJobStillRunning(await getBuildJob(job.jobId));
+      await assertJobChapterExists(document, job.chapterId);
       await assertCurrentBuildInputRevision(job, document);
       await replaceChapterFtsIndexArtifact(document, job.chapterId);
     });
@@ -376,11 +379,13 @@ async function executeIndexArtifactBuildJob(
         : "embedding-summary",
     sentences: snapshot.sentences,
     serialId: job.chapterId,
+    signal: context.signal,
     sourceRevision: snapshot.revision,
   });
 
   await new WikiGraphArchiveFile(job.archivePath).write(async (document) => {
     assertJobStillRunning(await getBuildJob(job.jobId));
+    await assertJobChapterExists(document, job.chapterId);
     await assertCurrentBuildInputRevision(job, document);
     await document.indexArtifacts.replaceEmbedding(artifact);
   });
@@ -407,7 +412,11 @@ async function readIndexArtifactJobSnapshot(job: BuildJob): Promise<{
 }> {
   return await new WikiGraphArchiveFile(job.archivePath).readDocument(
     async (document) => {
+      await assertJobChapterExists(document, job.chapterId);
       const revision = await document.serials.getRevision(job.chapterId);
+      if (job.target === "index-fts") {
+        return { revision, sentences: [] };
+      }
       const stream =
         job.target === "index-embedding-summary"
           ? document.getSummaryFragments(job.chapterId)
@@ -433,6 +442,19 @@ async function readIndexArtifactJobSnapshot(job: BuildJob): Promise<{
   );
 }
 
+async function assertJobChapterExists(
+  document: ReadonlyDocument,
+  chapterId: number,
+): Promise<void> {
+  if ((await document.serials.getById(chapterId)) !== undefined) {
+    return;
+  }
+
+  throw new Error(
+    `Chapter ${chapterId} no longer exists. Queue a new build job for an existing chapter.`,
+  );
+}
+
 export function requireKnowledgeGraphWikispineConfig(
   config: CLIConfig,
 ): NonNullable<CLIConfig["wikispine"]> {

packages/cli/src/runtime/embedding.ts

@@ -92,12 +92,16 @@ export function buildSearchIndexEmbeddingProvider(
       : { dimensions: config.dimensions }),
     identity: createEmbeddingIdentity(provider, model, config),
     model,
-    embedTexts: async (texts) => {
+    embedTexts: async (texts, options) => {
       const embeddings: number[][] = [];
       let tokens = 0;
 
       for (const batch of chunkTexts(texts, EMBEDDING_BATCH_SIZE)) {
+        options?.signal?.throwIfAborted();
         const result = await embedMany({
+          ...(options?.signal === undefined
+            ? {}
+            : { abortSignal: options.signal }),
           model: embeddingModel,
           ...(providerOptions === undefined ? {} : { providerOptions }),
           values: [...batch],

packages/core/data/help/topics/readiness.jinja

@@ -36,7 +36,7 @@ Archive index readiness:
     FTS artifact only: FTS query.
     Source embedding artifact only: semantic/Dense-only query.
     Both artifacts: Hybrid query with automatic fusion.
-    Missing both FTS and source embedding for any content chapter: ordinary query fails in strict mode.
+    Missing both FTS and source embedding for any content chapter: ordinary query fails.
     Summary embedding artifacts are for summary semantic retrieval and are not the minimum requirement for ordinary source query.
 
   Commands:

packages/core/src/library/search-index.ts

@@ -9,7 +9,9 @@ import { openSharedStateDatabase } from "../document/index.js";
 import { WikiGraphArchiveFile } from "../storage/wikg/index.js";
 import {
   assertArchiveIndexArtifactsReady,
-  buildArchiveIndexProjection,
+  readArchiveEmbeddingState,
+  readEmbeddingDimensions,
+  readEmbeddingModel,
   streamArchiveIndexProjection,
 } from "../retrieval/query/archive-view/index-state.js";
 import type {
@@ -478,11 +480,6 @@ async function replaceLibrarySearchIndex(
         async (archiveDocument) => {
           await assertArchiveIndexArtifactsReady(archiveDocument);
 
-          const archiveInput =
-            await buildArchiveIndexProjection(archiveDocument);
-          hasFts =
-            hasFts || archiveInput.textSentences.some((row) => row.text !== "");
-
           for await (const batch of streamArchiveIndexProjection(
             archiveDocument,
             archive.id,
@@ -491,6 +488,7 @@ async function replaceLibrarySearchIndex(
               const rowId = await insertTextSentenceRecord(database, record);
 
               if (record.text !== "") {
+                hasFts = true;
                 await insertFtsRecord(
                   database,
                   "text_sentence_fts",
@@ -604,39 +602,6 @@ function createLibraryIndexSearchFingerprint(
     .digest("hex");
 }
 
-async function readArchiveEmbeddingState(
-  document: Parameters<typeof assertArchiveIndexArtifactsReady>[0],
-): Promise<SearchIndexStoredEmbeddingState | undefined> {
-  let state: SearchIndexStoredEmbeddingState | undefined;
-
-  for (const artifactKind of [
-    "embedding-source",
-    "embedding-summary",
-  ] as const) {
-    for (const artifact of await document.indexArtifacts.list(artifactKind)) {
-      const dimensions = readEmbeddingDimensions(artifact.metadata);
-      const model = readEmbeddingModel(artifact.metadata);
-      const label = artifactKind === "embedding-source" ? "Source" : "Summary";
-
-      if (dimensions === undefined || model === undefined) {
-        throw new Error(
-          `${label} embedding artifact for chapter ${artifact.serialId} is missing embedding metadata.`,
-        );
-      }
-
-      state = mergeEmbeddingState(state, {
-        dimensions,
-        ...(readEmbeddingIdentity(artifact.metadata) === undefined
-          ? {}
-          : { identity: readEmbeddingIdentity(artifact.metadata)! }),
-        model,
-      });
-    }
-  }
-
-  return state;
-}
-
 function mergeEmbeddingState(
   current: SearchIndexStoredEmbeddingState | undefined,
   next: SearchIndexStoredEmbeddingState,
@@ -650,39 +615,13 @@ function mergeEmbeddingState(
     current.identity !== next.identity
   ) {
     throw new Error(
-      "Source embedding artifacts use different embedding providers or dimensions; rebuild them with one embeddings configuration.",
+      "Embedding artifacts use different embedding configurations; rebuild them with one embeddings configuration.",
     );
   }
 
   return current;
 }
 
-function readEmbeddingDimensions(
-  metadata: Readonly<Record<string, unknown>>,
-): number | undefined {
-  const value = metadata.dimensions;
-
-  return Number.isInteger(value) && Number(value) > 0
-    ? Number(value)
-    : undefined;
-}
-
-function readEmbeddingIdentity(
-  metadata: Readonly<Record<string, unknown>>,
-): string | undefined {
-  const value = metadata.identity;
-
-  return typeof value === "string" && value !== "" ? value : undefined;
-}
-
-function readEmbeddingModel(
-  metadata: Readonly<Record<string, unknown>>,
-): string | undefined {
-  const value = metadata.model;
-
-  return typeof value === "string" && value !== "" ? value : undefined;
-}
-
 function formatLibraryHitSource(
   sourceByArchiveId: ReadonlyMap<number, WikiGraphLibraryIndexSource>,
   archiveId: number,

packages/core/src/retrieval/index-artifact/build.ts

@@ -206,6 +206,7 @@ export async function createEmbeddingIndexArtifactInput(input: {
   readonly kind: EmbeddingIndexArtifactKind;
   readonly sentences: readonly SentenceRecord[];
   readonly serialId: number;
+  readonly signal?: AbortSignal;
   readonly sourceRevision: number;
 }): Promise<ReplaceEmbeddingIndexArtifactInput> {
   const segments = createEmbeddingSegments(input.sentences);
@@ -215,6 +216,7 @@ export async function createEmbeddingIndexArtifactInput(input: {
       : (
           await input.embeddingProvider.embedTexts(
             segments.map((segment) => segment.text),
+            input.signal === undefined ? undefined : { signal: input.signal },
           )
         ).embeddings;
 
@@ -358,11 +360,12 @@ function collectTocItems(items: readonly TocItem[]): readonly TocItem[] {
 function createEmbeddingSegments(
   sentences: readonly SentenceRecord[],
 ): readonly Omit<IndexArtifactEmbeddingSegment, "vector">[] {
+  assertDenseSegmentConstants();
   const records = sentences
     .map((sentence, sentenceIndex) => ({
       sentenceIndex,
       text: sentence.text,
-      wordsCount: sentence.wordsCount,
+      wordsCount: requireNonNegativeWordsCount(sentence.wordsCount),
     }))
     .filter((record) => record.text.trim() !== "");
   const segments: Omit<IndexArtifactEmbeddingSegment, "vector">[] = [];
@@ -389,11 +392,6 @@ function createEmbeddingSegments(
       }
     }
 
-    if (end <= start) {
-      end = start + 1;
-      wordsCount = Math.max(0, records[start]!.wordsCount);
-    }
-
     const segmentRecords = records.slice(start, end);
     const segment = createEmbeddingSegment(segmentRecords, segments.length);
 
@@ -425,6 +423,25 @@ function createEmbeddingSegments(
   }));
 }
 
+function assertDenseSegmentConstants(): void {
+  if (
+    DENSE_SEGMENT_MIN_WORDS < 0 ||
+    DENSE_SEGMENT_OVERLAP_WORDS < 0 ||
+    DENSE_SEGMENT_TARGET_WORDS < DENSE_SEGMENT_MIN_WORDS ||
+    DENSE_SEGMENT_MAX_WORDS < DENSE_SEGMENT_TARGET_WORDS
+  ) {
+    throw new Error("Invalid Dense segment word-count configuration.");
+  }
+}
+
+function requireNonNegativeWordsCount(wordsCount: number): number {
+  if (!Number.isFinite(wordsCount) || wordsCount < 0) {
+    throw new Error("Sentence word count must be non-negative.");
+  }
+
+  return wordsCount;
+}
+
 function createEmbeddingSegment(
   records: readonly {
     readonly sentenceIndex: number;

packages/core/src/retrieval/query/archive-view/index-state.ts

@@ -516,7 +516,7 @@ function countSearchIndexBatchRecords(batch: SearchIndexWriteBatch): number {
   return batch.objectProperties.length + batch.textSentences.length;
 }
 
-async function readArchiveEmbeddingState(
+export async function readArchiveEmbeddingState(
   document: ReadonlyDocument,
 ): Promise<SearchIndexStoredEmbeddingState | undefined> {
   let state: SearchIndexStoredEmbeddingState | undefined;
@@ -550,7 +550,7 @@ async function readArchiveEmbeddingState(
           state.identity !== next.identity)
       ) {
         throw new Error(
-          "Embedding artifacts use different embedding providers or dimensions; rebuild them with one embeddings configuration.",
+          "Embedding artifacts use different embedding configurations; rebuild them with one embeddings configuration.",
         );
       }
       state = next;
@@ -566,7 +566,7 @@ function readWordsCount(metadata: Readonly<Record<string, unknown>>): number {
   return typeof value === "number" && Number.isFinite(value) ? value : 0;
 }
 
-function readEmbeddingDimensions(
+export function readEmbeddingDimensions(
   metadata: Readonly<Record<string, unknown>>,
 ): number | undefined {
   const value = metadata.dimensions;
@@ -576,15 +576,15 @@ function readEmbeddingDimensions(
     : undefined;
 }
 
-function readEmbeddingIdentity(
+export function readEmbeddingIdentity(
   metadata: Readonly<Record<string, unknown>>,
 ): string | undefined {
   const value = metadata.identity;
 
   return typeof value === "string" && value !== "" ? value : undefined;
 }
 
-function readEmbeddingModel(
+export function readEmbeddingModel(
   metadata: Readonly<Record<string, unknown>>,
 ): string | undefined {
   const value = metadata.model;