feat(compile): allow callers to skip embeddings

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- **Skip embeddings on compile** — `compile({ embeddings: false })` runs page generation, links and the lexical index without any embedding-provider call or pending-embedding retry, for an SDK host that maintains its own semantic index. Omitting it is unchanged.
+
+  The flag closes a gap that is provider-specific. With `anthropic` or `claude-agent` a caller can already opt out by not setting `VOYAGE_API_KEY`, but the `openai` embedding credential falls back to `OPENAI_API_KEY` and `ollama` needs no key at all, so those callers had no way to compile without embedding except to break chat.
+
+  Contributed by **@TigerOfCountryYao** (#169).
+
 - **Caller system policy on compile** — `compile({ systemPolicy })` appends deployment-specific editorial or publication guidance to the built-in compile prompts, for SDK hosts that need it without forking the prompts. It is additive rather than a replacement, sits before the source material, and blank or omitted leaves the prompt byte-identical.
 
   It is advisory rather than enforceable: a policy makes a model more likely to follow a rule, and nothing downstream verifies that it did. Anything that must hold belongs in a lint rule or a trust gate.

docs/guides/sdk.mdx

@@ -85,6 +85,7 @@ Both methods return `IngestResult` with `filename`, `chars`, and `truncated` fie
     It is **advisory, not enforceable**. A policy makes the model more likely to follow a rule; it cannot make it obey one, and nothing verifies that it did. Anything that must hold belongs in a lint rule or a trust gate.
 
     Changing it **regenerates the pages compiled under the previous policy**, the same way changing the output language does, so it costs a full compile of the affected pages. Each page records the policy's digest (never its text) in `promptModifiers`.
+  - `options.embeddings` - set to `false` to skip embedding generation and pending-embedding retries. Page generation, links, and the lexical index still run.
 
   Source content is sent to the configured LLM provider during compilation. Do not compile wikis containing confidential data unless the provider's data-handling policies are acceptable for that content.
 

src/compiler/index.ts

@@ -358,6 +358,7 @@ async function seedThenFinalize(
   await maybeSeedPages(root, schema, generation, options);
   await finalizeWiki(root, draft, generation.writtenPages, generation.seedSlugs, {
     scoped: options.changeFilter !== undefined,
+    embeddings: options.embeddings !== false,
   });
 }
 
@@ -546,6 +547,16 @@ async function markDeletedAsOrphaned(
  * them rather than to add a sixth parameter.
  */
 interface FinalizeFlags {
+  /**
+   * Refresh the embedding store after the durable state write. `false` skips it
+   * entirely, including the pending-embeddings drain, for a caller that keeps
+   * its own semantic index.
+   *
+   * Not a prompt modifier: it changes what runs AFTER pages are written, never
+   * what any prompt asks for, so it must not enter the modifier digest or
+   * invalidate a page that is byte-identical under it.
+   */
+  embeddings?: boolean;
   /**
    * The run recompiled a SUBSET by design (`refresh --stale` supplies a
    * `changeFilter`), so it must not record the prompt-modifier selection as
@@ -569,7 +580,7 @@ async function finalizeWiki(
   seedSlugs: string[] = [],
   flags: FinalizeFlags = {},
 ): Promise<void> {
-  const { scoped = false } = flags;
+  const { scoped = false, embeddings = true } = flags;
   const conceptChangedSlugs = pages.map((entry) => entry.slug);
   const conceptNewSlugs = pages
     .filter((entry) => entry.concept.is_new)
@@ -608,7 +619,7 @@ async function finalizeWiki(
 
   await generateIndex(root);
   await generateMOC(root);
-  await safelyUpdateEmbeddings(root, allChangedSlugs);
+  if (embeddings) await safelyUpdateEmbeddings(root, allChangedSlugs);
 }
 
 /**

src/sdk/types.ts

@@ -97,6 +97,11 @@ export interface SdkCompileOptions {
    * way changing the output language does.
    */
   systemPolicy?: string;
+  /**
+   * Refresh semantic embeddings after compilation. Defaults to true.
+   * False prevents embedding-provider calls and pending-embedding retries.
+   */
+  embeddings?: boolean;
 }
 
 /** Options for `getContextPack`. Maps onto the subset of BuildContextPackOptions needed externally. */

src/utils/types.ts

@@ -218,6 +218,12 @@ export interface CompileOptions {
    * policy, and its digest is recorded per page. See compiler/prompt-modifiers.ts.
    */
   systemPolicy?: string;
+  /**
+   * Refresh semantic embeddings after compilation. Defaults to true.
+   * Set to false for a lexical-only build with no embedding-provider calls or
+   * pending-embedding retries.
+   */
+  embeddings?: boolean;
 }
 
 /**

test/compile-options.test.ts

@@ -0,0 +1,67 @@
+/**
+ * @file test/compile-options.test.ts
+ * @description Public compile-option coverage for embedding suppression.
+ */
+
+import { describe, expect, it, vi } from "vitest";
+import { createWiki } from "../src/sdk/wiki.js";
+import { AnthropicProvider } from "../src/providers/anthropic.js";
+import * as embeddings from "../src/utils/embeddings.js";
+import { useCompileProject } from "./fixtures/compile-project.js";
+
+const EXTRACTION = JSON.stringify({
+  concepts: [{ concept: "Alpha", summary: "Alpha summary.", is_new: true }],
+});
+
+const ctx = useCompileProject({
+  dirSuffix: "options",
+  sourceFile: "sample.md",
+  sourceContent: "# Alpha\n\nAlpha is documented here.",
+});
+
+/** Stub a one-concept compile and suppress terminal noise. */
+function stubCompile(): void {
+  vi.spyOn(AnthropicProvider.prototype, "toolCall").mockResolvedValue(EXTRACTION);
+  vi.spyOn(AnthropicProvider.prototype, "complete").mockResolvedValue("Alpha body. ^[sample.md]");
+  vi.spyOn(console, "log").mockImplementation(() => {});
+}
+
+describe("compile options", () => {
+  it("embeddings:false skips embedding refresh while still compiling pages", async () => {
+    stubCompile();
+    const embedSpy = vi
+      .spyOn(embeddings, "updateEmbeddingsLockedCore")
+      .mockResolvedValue({ embedded: [], eligible: [] });
+
+    const result = await createWiki({ root: ctx.dir }).compile({ embeddings: false });
+
+    expect(result.pages).toContain("alpha");
+    expect(embedSpy).not.toHaveBeenCalled();
+  });
+
+  /** Compile with `options`, returning whether the embedding refresh ran. */
+  async function compileAndWatchEmbeddings(
+    options?: Parameters<ReturnType<typeof createWiki>["compile"]>[0],
+  ): Promise<{ refreshed: boolean; pages: string[] }> {
+    stubCompile();
+    const embedSpy = vi
+      .spyOn(embeddings, "updateEmbeddingsLockedCore")
+      .mockResolvedValue({ embedded: [], eligible: [] });
+    const result = await createWiki({ root: ctx.dir }).compile(options);
+    return { refreshed: embedSpy.mock.calls.length > 0, pages: result.pages };
+  }
+
+  // The negative case alone cannot distinguish "the flag works" from
+  // "embeddings are broken for everyone": a build that never refreshes
+  // satisfies it. The default path is what the flag promises to leave alone,
+  // so these are what give the case above its meaning.
+  it("still refreshes embeddings when the option is omitted", async () => {
+    const run = await compileAndWatchEmbeddings();
+    expect(run.pages).toContain("alpha");
+    expect(run.refreshed).toBe(true);
+  });
+
+  it("still refreshes embeddings when the option is explicitly true", async () => {
+    expect((await compileAndWatchEmbeddings({ embeddings: true })).refreshed).toBe(true);
+  });
+});