diff --git a/apps/host-cloudflare/src/plugins.ts b/apps/host-cloudflare/src/plugins.ts index 3f31792a2..194115e6f 100644 --- a/apps/host-cloudflare/src/plugins.ts +++ b/apps/host-cloudflare/src/plugins.ts @@ -14,6 +14,7 @@ import { serviceTokensPlugin } from "@executor-js/plugin-service-tokens/server"; import { semanticSearchHttpPlugin } from "@executor-js/plugin-semantic-search/api"; import { makeVectorizeStore, + ToolSearchBackend, withCloudflareLimits, type VectorizeIndex, } from "@executor-js/plugin-semantic-search"; @@ -51,6 +52,14 @@ export const makeCloudflarePlugins = ( searchNamespace?: string, ) => { const store = vectorize ? withCloudflareLimits(makeVectorizeStore(vectorize)) : undefined; + const semanticSearchBackend = + store && geminiApiKey + ? ToolSearchBackend.vector({ + store, + geminiApiKey, + namespace: searchNamespace, + }) + : undefined; return [ openApiHttpPlugin(), googleHttpPlugin(), @@ -62,7 +71,7 @@ export const makeCloudflarePlugins = ( observer: () => (analytics ? createWaeMetricsObserver(analytics) : noopExecutionObserver), }), serviceTokensPlugin(), - semanticSearchHttpPlugin({ store, geminiApiKey, namespace: searchNamespace }), + semanticSearchHttpPlugin({ backend: semanticSearchBackend }), ] as const; }; diff --git a/packages/plugins/semantic-search/src/sdk/index.ts b/packages/plugins/semantic-search/src/sdk/index.ts index 29aef276b..2ef1b26f3 100644 --- a/packages/plugins/semantic-search/src/sdk/index.ts +++ b/packages/plugins/semantic-search/src/sdk/index.ts @@ -3,6 +3,17 @@ export { type SemanticSearchPluginOptions, type SemanticSearchExtension, } from "./plugin"; +export { + ToolSearchBackend, + makeVectorToolSearchBackend, + type SemanticSearchRefreshResult, + type SemanticSearchResultPage, + type SemanticSearchStatus, + type ToolSearchBackend as ToolSearchBackendType, + type ToolSearchBackendFactory, + type VectorToolSearchBackendStorage, + type VectorToolSearchBackendOptions, +} from "./tool-search-backend"; export { makeVectorToolDiscoveryProvider } from "./provider"; export { makeGeminiEmbedder, diff --git a/packages/plugins/semantic-search/src/sdk/plugin.test.ts b/packages/plugins/semantic-search/src/sdk/plugin.test.ts index e5da65226..cd3b4d752 100644 --- a/packages/plugins/semantic-search/src/sdk/plugin.test.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.test.ts @@ -3,8 +3,9 @@ import { Effect } from "effect"; import type { ToolDiscoveryProvider, ToolDiscoveryResult } from "@executor-js/sdk/core"; -import { makeWholeChunker } from "./chunker"; -import { makeSemanticSearchExtension } from "./plugin"; +import { SemanticSearchError } from "./errors"; +import { makeSemanticSearchExtension, semanticSearchPlugin } from "./plugin"; +import type { ToolSearchBackend, ToolSearchBackendFactory } from "./tool-search-backend"; // A result whose integration/path is NOT the tenant id. This is exactly the case // the bug broke: the operator search defaulted the integration-prefix filter to @@ -23,18 +24,33 @@ const githubItem: ToolDiscoveryResult = { // by `search` (they drive indexing), so they stay undefined. const extensionWith = (provider: ToolDiscoveryProvider) => makeSemanticSearchExtension({ - namespace: "default", // tenant id — NOT a tool prefix - embedder: undefined, - store: undefined, - chunker: makeWholeChunker(), - fingerprints: undefined, - indexRuns: undefined, - indexJobs: undefined, - indexChunks: undefined, - blobs: undefined, - owner: undefined, - lexicalStore: undefined, - provider, + backend: { + namespace: "default", // tenant id — NOT a tool prefix + provider, + index: () => undefined as never, + reindex: () => Effect.die("unused"), + sweep: () => Effect.die("unused"), + search: (executor, input) => + provider + .searchTools({ + executor, + query: input.query, + namespace: input.namespace, + limit: input.limit ?? 20, + offset: 0, + }) + .pipe( + Effect.map((page) => ({ + namespace: "default", + query: input.query, + items: page.items, + })), + Effect.mapError( + (cause) => new SemanticSearchError({ message: "Test provider failed.", cause }), + ), + ), + status: () => Effect.die("unused"), + }, }); describe("makeSemanticSearchExtension — search namespace handling", () => { @@ -83,3 +99,37 @@ describe("makeSemanticSearchExtension — search namespace handling", () => { }), ); }); + +describe("semanticSearchPlugin — backend storage", () => { + it("registers and builds storage through the selected backend", () => { + const pluginStorage = {}; + const backendStorage = { marker: true }; + let receivedStorage: unknown; + + const backend: ToolSearchBackend = { + namespace: "default", + index: () => undefined as never, + reindex: () => Effect.die("unused"), + sweep: () => Effect.die("unused"), + search: () => Effect.die("unused"), + status: () => Effect.die("unused"), + }; + const factory: ToolSearchBackendFactory = { + namespace: "default", + pluginStorage, + storage: () => backendStorage, + build: ({ storage }) => { + receivedStorage = storage; + return backend; + }, + }; + + const plugin = semanticSearchPlugin({ backend: factory }); + const storage = plugin.storage(undefined as never); + plugin.extension?.({ storage } as never); + + expect(plugin.pluginStorage).toBe(pluginStorage); + expect(storage.backend).toBe(backendStorage); + expect(receivedStorage).toBe(backendStorage); + }); +}); diff --git a/packages/plugins/semantic-search/src/sdk/plugin.ts b/packages/plugins/semantic-search/src/sdk/plugin.ts index 47f9c0bfd..442788d9e 100644 --- a/packages/plugins/semantic-search/src/sdk/plugin.ts +++ b/packages/plugins/semantic-search/src/sdk/plugin.ts @@ -1,350 +1,60 @@ -import { - definePlugin, - type Executor, - type ToolDiscoveryProvider, - type ToolDiscoveryResult, -} from "@executor-js/sdk/core"; -import { Effect } from "effect"; +import { definePlugin, type Executor } from "@executor-js/sdk/core"; +import type { Effect } from "effect"; -import { type Chunker, makeFacetChunker } from "./chunker"; -import { indexChunks, indexJobs, indexRuns, toolFingerprints } from "./collections"; -import { makeGeminiEmbedder, type ToolEmbedder } from "./embedder"; -import { SemanticSearchError } from "./errors"; -import { makeHybridToolDiscoveryProvider } from "./hybrid"; +import type { SemanticSearchError } from "./errors"; import { - make as makeToolSearchIndex, - run as runToolSearchIndex, - sweepRemoved, - type ToolSearchIndex, -} from "./tool-search-index"; -import { makeVectorToolDiscoveryProvider } from "./provider"; -import type { VectorStore } from "./store"; -import { type FtsLexicalStore, makeFtsLexicalProvider } from "./store-fts"; + notConfigured, + type SemanticSearchResultPage, + type SemanticSearchStatus, + type ToolSearchBackend, + type ToolSearchBackendFactory, + unconfiguredIndex, +} from "./tool-search-backend"; export interface SemanticSearchPluginOptions { - /** A vector store — construct one with `makeVectorizeStore` (Cloudflare), - * `makeZVecStore` (local/dev), or any other `VectorStore` implementation. - * Absent → the plugin is inert and the engine keeps its built-in lexical - * search, mirroring how the metrics plugin no-ops without its binding. */ - readonly store?: VectorStore; - /** Gemini API key (a wrangler secret on the Cloudflare host). Absent → inert, - * unless a custom `embedder` is supplied. */ - readonly geminiApiKey?: string; - /** Namespace isolating this tenant's vectors. The single-org self-host passes - * its org id; multi-tenant per-request scoping is a follow-up. Defaults to - * `"default"`. */ - readonly namespace?: string; - /** Gemini embedding model id. Defaults to the v2 model (see embedder). */ - readonly model?: string; - /** Embedding dimensionality — MUST equal the vector index's dimensions. */ - readonly dimensions?: number; - /** Gemini embedding batch size (texts per request; the Google provider allows - * up to 2048 values per call, but Workers memory is the practical ceiling). - * Larger batches - * mean fewer, fatter requests — lower peak RPM for the same tokens — which is - * the key lever when a reindex fans out across many concurrent workers and is - * request-rate-bound. Defaults to the embedder's own default. */ - readonly embedderBatchSize?: number; - /** Inject a custom embedder (tests). Overrides `model`/`geminiApiKey`/`dimensions`. */ - readonly embedder?: ToolEmbedder; - /** Chunker used when indexing the tool catalog. Defaults to `makeFacetChunker()`. - * Override in tests or to benchmark the whole chunker. */ - readonly chunker?: Chunker; - /** Lexical discovery provider to fuse with the vector provider (RRF). The host - * supplies the engine's built-in `defaultToolDiscoveryProvider` here — kept as - * an option so the plugin never depends on `@executor-js/execution`. Absent → - * the plugin replaces tool search with the vector provider ALONE (the engine's - * lexical scorer does NOT run, because a plugin provider supersedes it). Supply - * it to get hybrid lexical+vector search. */ - readonly lexical?: ToolDiscoveryProvider; - /** A populated FTS5 lexical store — e.g. `makeD1FtsLexicalStore(env.DB)` on - * Cloudflare, or `makeFtsLexicalStore({ path })` locally. When supplied, - * `reindex` ALSO writes each tool's lexical document here, and the store is - * wrapped as the hybrid `lexical` provider — giving real FTS5/BM25 + vector - * RRF without the host owning lexical indexing. Takes precedence over - * `lexical`. */ - readonly lexicalStore?: FtsLexicalStore; -} - -const notConfigured = (): Effect.Effect => - Effect.fail( - new SemanticSearchError({ - message: "Semantic search is not configured (missing the vector store or Gemini API key).", - }), - ); - -/** Default page size for the operator `search` surface. */ -const DEFAULT_SEARCH_LIMIT = 20; -const DEFAULT_IN_PROCESS_PARTITIONS = 1; - -/** A live `tools.search` result page from the operator search surface. */ -export interface SemanticSearchResultPage { - readonly namespace: string; - readonly query: string; - readonly items: readonly ToolDiscoveryResult[]; -} - -/** Operator-facing index status: indexed (vector) + lexical document counts. */ -export interface SemanticSearchStatus { - readonly namespace: string; - /** Tools with a stored fingerprint — the vector-indexed count. */ - readonly indexed: number; - /** FTS5 lexical documents, or `null` when no lexical store is configured. */ - readonly lexical: number | null; + readonly backend?: ToolSearchBackendFactory; } -/** Build the `executor.semanticSearch` surface: `reindex` (reconcile the catalog - * into the vector + lexical index), `search` (live `tools.search` through the - * shared provider), and `status` (index counts). `reindex`/`search` take the - * scoped executor because only the request/API layer holds it. `reindex`/`search` - * are inert — failing clearly — until both a vector store and an embedder are - * present. */ -// Exported for unit testing the `search` namespace handling; not re-exported -// from the package index (see sdk/index.ts), so the public API is unchanged. export const makeSemanticSearchExtension = (deps: { - readonly namespace: string; - readonly embedder: ToolEmbedder | undefined; - readonly store: VectorStore | undefined; - readonly chunker: Chunker; - readonly fingerprints: Parameters[0]["fingerprints"] | undefined; - readonly indexRuns: Parameters[0]["runs"] | undefined; - readonly indexJobs: Parameters[0]["jobs"] | undefined; - readonly indexChunks: Parameters[0]["chunks"] | undefined; - readonly blobs: Parameters[0]["blobs"] | undefined; - readonly owner: Parameters[0]["owner"] | undefined; - readonly lexicalStore: FtsLexicalStore | undefined; - readonly provider: ToolDiscoveryProvider | undefined; -}) => { - const unconfiguredIndex: ToolSearchIndex.Service = { - create: () => notConfigured(), - scan: () => notConfigured(), - chunk: () => notConfigured(), - embed: () => notConfigured(), - commit: () => notConfigured(), - fail: () => notConfigured(), - reconcile: () => notConfigured(), - status: () => notConfigured(), - complete: () => notConfigured(), - }; - const index = (executor: Executor): ToolSearchIndex.Service => - deps.embedder && - deps.store && - deps.fingerprints && - deps.indexRuns && - deps.indexJobs && - deps.indexChunks && - deps.blobs && - deps.owner - ? makeToolSearchIndex({ - namespace: deps.namespace, - executor, - embedder: deps.embedder, - store: deps.store, - chunker: deps.chunker, - runs: deps.indexRuns, - jobs: deps.indexJobs, - chunks: deps.indexChunks, - fingerprints: deps.fingerprints, - blobs: deps.blobs, - owner: deps.owner, - lexicalStore: deps.lexicalStore, - }) - : unconfiguredIndex; - - return { - index, - reindex: (executor: Executor): Effect.Effect => - deps.embedder && - deps.store && - deps.fingerprints && - deps.indexRuns && - deps.indexJobs && - deps.indexChunks && - deps.blobs && - deps.owner - ? runToolSearchIndex({ - namespace: deps.namespace, - executor, - embedder: deps.embedder, - store: deps.store, - chunker: deps.chunker, - runs: deps.indexRuns, - jobs: deps.indexJobs, - chunks: deps.indexChunks, - fingerprints: deps.fingerprints, - blobs: deps.blobs, - owner: deps.owner, - lexicalStore: deps.lexicalStore, - runId: `manual-${Date.now()}`, - partitionCount: DEFAULT_IN_PROCESS_PARTITIONS, - }) - : notConfigured(), - - /** Delete index entries for tools that left the catalog. Needs no embedder. */ - sweep: ( - executor: Executor, - ): Effect.Effect< - { readonly namespace: string; readonly removed: number }, - SemanticSearchError - > => - deps.store && deps.fingerprints && deps.blobs && deps.owner - ? sweepRemoved({ - namespace: deps.namespace, - executor, - store: deps.store, - fingerprints: deps.fingerprints, - blobs: deps.blobs, - owner: deps.owner, - lexicalStore: deps.lexicalStore, - }) - : notConfigured(), - - /** Run a live `tools.search` through the same provider the engine uses, so the - * operator console sees exactly what the agent would. Inert until configured. */ - search: ( - executor: Executor, - input: { readonly query: string; readonly namespace?: string; readonly limit?: number }, - ): Effect.Effect => { - // `input.namespace` is an OPTIONAL integration/path-prefix filter — distinct - // from the tenant `deps.namespace` (Vectorize storage isolation, applied - // inside the provider's store.query). It must NOT default to the tenant - // namespace: a tenant id like "default" is not a tool prefix, so the - // provider's `matchesNamespace` would drop every result. Absent → no prefix - // filter; the response still reports the tenant namespace. - const prefix = input.namespace; - return deps.provider - ? deps.provider - .searchTools({ - executor, - query: input.query, - namespace: prefix, - limit: input.limit ?? DEFAULT_SEARCH_LIMIT, - offset: 0, - }) - .pipe( - Effect.map((page) => ({ - namespace: deps.namespace, - query: input.query, - items: page.items, - })), - Effect.mapError( - (cause) => - new SemanticSearchError({ message: "Semantic search query failed.", cause }), - ), - ) - : notConfigured(); - }, - - /** Index status for the operator console: vector (fingerprint) + lexical counts. */ - status: (): Effect.Effect => - Effect.gen(function* () { - const indexed = deps.fingerprints - ? yield* deps.fingerprints - .count() - .pipe( - Effect.mapError( - (cause) => - new SemanticSearchError({ message: "Failed to count indexed tools.", cause }), - ), - ) - : 0; - // A transient lexical-count failure must not mask the (already-fetched) - // vector count: degrade `lexical` to null and still return `indexed`. - const lexical = deps.lexicalStore - ? yield* deps.lexicalStore - .count(deps.namespace) - .pipe(Effect.catch(() => Effect.succeed(null))) - : null; - return { namespace: deps.namespace, indexed, lexical }; - }), - }; -}; + readonly backend: ToolSearchBackend | undefined; +}) => ({ + index: (executor: Executor) => deps.backend?.index(executor) ?? unconfiguredIndex, + reindex: (executor: Executor) => deps.backend?.reindex(executor) ?? notConfigured(), + sweep: (executor: Executor) => deps.backend?.sweep(executor) ?? notConfigured(), + search: ( + executor: Executor, + input: { readonly query: string; readonly namespace?: string; readonly limit?: number }, + ): Effect.Effect => + deps.backend?.search(executor, input) ?? notConfigured(), + status: (): Effect.Effect => + deps.backend?.status() ?? notConfigured(), + provider: deps.backend?.provider, +}); /** The `executor.semanticSearch` surface, derived from its factory. */ export type SemanticSearchExtension = ReturnType; /** - * Semantic `tools.search` backed by a vector store + Gemini embeddings. - * Supplies a `runtime.toolDiscoveryProvider`, so it supersedes the engine's - * built-in lexical scorer wherever it is registered. The tool catalog is - * indexed explicitly via the `reindex` extension method (see the `/api` subpath - * for the HTTP route). - * - * When both an embedder and a vector store are present, the plugin exposes a - * discovery provider. If the host also supplies `lexical`, that provider and the - * vector provider are fused with Reciprocal Rank Fusion (hybrid search); - * otherwise the vector provider answers alone. + * Semantic `tools.search` backed by a host-selected search backend. + * The backend owns indexing, querying, and optional runtime discovery. When it + * exposes a `provider`, the engine uses it in place of the built-in lexical + * scorer. Without a backend, the plugin remains registered for stable API shape + * and fails explicit semantic-search calls with a typed configuration error. */ export const semanticSearchPlugin = definePlugin((options?: SemanticSearchPluginOptions) => { - const namespace = options?.namespace ?? "default"; - const embedder = - options?.embedder ?? - (options?.geminiApiKey - ? makeGeminiEmbedder({ - apiKey: options.geminiApiKey, - model: options.model, - dimensions: options.dimensions, - batchSize: options.embedderBatchSize, - }) - : undefined); - // Inert without both a vector store and an embedder — the engine then - // keeps its built-in lexical search (mirrors the metrics plugin's no-op). - const store = options?.store; - const chunker = options?.chunker ?? makeFacetChunker(); - const lexical = options?.lexical; - const lexicalStore = options?.lexicalStore; - - // Build the discovery provider once and share it between the engine's - // `tools.search` (runtime) and the operator `search` extension surface, so - // both answer through exactly the same vector/hybrid path. A populated FTS5 - // store (wrapped as a provider) takes precedence over a host-supplied lexical - // provider; either side fuses with the vector provider via RRF. - const provider: ToolDiscoveryProvider | undefined = - !embedder || !store - ? undefined - : (() => { - const vector = makeVectorToolDiscoveryProvider({ embedder, store, namespace }); - const lexicalProvider = lexicalStore - ? makeFtsLexicalProvider(lexicalStore, namespace) - : lexical; - return lexicalProvider - ? makeHybridToolDiscoveryProvider({ lexical: lexicalProvider, vector }) - : vector; - })(); - return { id: "semanticSearch" as const, packageName: "@executor-js/plugin-semantic-search", - pluginStorage: { toolFingerprints, indexRuns, indexJobs, indexChunks }, + pluginStorage: options?.backend?.pluginStorage, storage: (deps) => ({ - fingerprints: deps.pluginStorage.collection(toolFingerprints), - indexRuns: deps.pluginStorage.collection(indexRuns), - indexJobs: deps.pluginStorage.collection(indexJobs), - indexChunks: deps.pluginStorage.collection(indexChunks), - indexBlobs: deps.blobs, - // The tool catalog is an org-level artifact, so fingerprints are ALWAYS - // org-scoped. Scoping by the triggering principal (user vs cron) would - // split the fingerprint store into disjoint partitions, so each reindex - // would see an empty store and re-embed the whole catalog. - owner: "org" as const, + backend: options?.backend?.storage(deps), }), extension: (ctx) => makeSemanticSearchExtension({ - namespace, - embedder, - store, - chunker, - fingerprints: ctx.storage.fingerprints, - indexRuns: ctx.storage.indexRuns, - indexJobs: ctx.storage.indexJobs, - indexChunks: ctx.storage.indexChunks, - blobs: ctx.storage.indexBlobs, - owner: ctx.storage.owner, - lexicalStore, - provider, + backend: options?.backend?.build({ storage: ctx.storage.backend }), }), runtime: { - toolDiscoveryProvider: () => provider, + toolDiscoveryProvider: (extension: SemanticSearchExtension) => extension.provider, }, }; }); diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-backend.ts b/packages/plugins/semantic-search/src/sdk/tool-search-backend.ts new file mode 100644 index 000000000..2acc55273 --- /dev/null +++ b/packages/plugins/semantic-search/src/sdk/tool-search-backend.ts @@ -0,0 +1,277 @@ +import { + type Executor, + type PluginBlobStore, + type PluginStorageConfig, + type PluginStorageCollectionFacade, + type StorageDeps, + type ToolDiscoveryProvider, + type ToolDiscoveryResult, +} from "@executor-js/sdk/core"; +import { Effect } from "effect"; + +import { type Chunker, makeFacetChunker } from "./chunker"; +import { indexChunks, indexJobs, indexRuns, toolFingerprints } from "./collections"; +import { makeGeminiEmbedder, type GeminiEmbedderOptions, type ToolEmbedder } from "./embedder"; +import { SemanticSearchError } from "./errors"; +import { makeHybridToolDiscoveryProvider } from "./hybrid"; +import { makeVectorToolDiscoveryProvider } from "./provider"; +import type { VectorStore } from "./store"; +import { type FtsLexicalStore, makeFtsLexicalProvider } from "./store-fts"; +import { + make as makeToolSearchIndex, + run as runToolSearchIndex, + sweepRemoved, + type ToolSearchIndex, +} from "./tool-search-index"; + +export interface SemanticSearchResultPage { + readonly namespace: string; + readonly query: string; + readonly items: readonly ToolDiscoveryResult[]; +} + +export interface SemanticSearchStatus { + readonly namespace: string; + readonly indexed: number; + readonly lexical: number | null; +} + +export interface SemanticSearchRefreshResult { + readonly namespace: string; + readonly total: number; + readonly indexed: number; + readonly skipped: number; + readonly removed: number; +} + +export interface ToolSearchBackend { + readonly namespace: string; + readonly provider?: ToolDiscoveryProvider; + readonly index: (executor: Executor) => ToolSearchIndex.Service; + readonly reindex: ( + executor: Executor, + ) => Effect.Effect; + readonly sweep: (executor: Executor) => Effect.Effect< + { + readonly namespace: string; + readonly removed: number; + }, + SemanticSearchError + >; + readonly search: ( + executor: Executor, + input: { readonly query: string; readonly namespace?: string; readonly limit?: number }, + ) => Effect.Effect; + readonly status: () => Effect.Effect; +} + +const DEFAULT_SEARCH_LIMIT = 20; +const DEFAULT_IN_PROCESS_PARTITIONS = 1; + +export interface VectorToolSearchBackendStorage { + readonly fingerprints: PluginStorageCollectionFacade; + readonly indexRuns: PluginStorageCollectionFacade; + readonly indexJobs: PluginStorageCollectionFacade; + readonly indexChunks: PluginStorageCollectionFacade; + readonly indexBlobs: PluginBlobStore; + readonly owner: "org" | "user"; +} + +export interface ToolSearchBackendFactory { + readonly namespace: string; + readonly pluginStorage?: PluginStorageConfig; + readonly storage: (deps: StorageDeps) => TStorage; + build(input: { readonly storage: TStorage }): ToolSearchBackend; +} + +export interface VectorToolSearchBackendOptions { + readonly namespace?: string; + readonly store: VectorStore; + readonly geminiApiKey?: string; + readonly model?: string; + readonly dimensions?: number; + readonly embedderBatchSize?: number; + readonly embedder?: ToolEmbedder; + readonly chunker?: Chunker; + readonly lexical?: ToolDiscoveryProvider; + readonly lexicalStore?: FtsLexicalStore; +} + +export const notConfigured = (): Effect.Effect => + Effect.fail( + new SemanticSearchError({ + message: "Semantic search is not configured (missing a tool-search backend).", + }), + ); + +export const unconfiguredIndex: ToolSearchIndex.Service = { + create: () => notConfigured(), + scan: () => notConfigured(), + chunk: () => notConfigured(), + embed: () => notConfigured(), + commit: () => notConfigured(), + fail: () => notConfigured(), + reconcile: () => notConfigured(), + status: () => notConfigured(), + complete: () => notConfigured(), +}; + +const makeVectorEmbedder = (options: VectorToolSearchBackendOptions): ToolEmbedder | undefined => + options.embedder ?? + (options.geminiApiKey + ? makeGeminiEmbedder({ + apiKey: options.geminiApiKey, + model: options.model, + dimensions: options.dimensions, + batchSize: options.embedderBatchSize, + } satisfies GeminiEmbedderOptions) + : undefined); + +const makeVectorProvider = (input: { + readonly namespace: string; + readonly embedder: ToolEmbedder | undefined; + readonly store: VectorStore; + readonly lexical?: ToolDiscoveryProvider; + readonly lexicalStore?: FtsLexicalStore; +}): ToolDiscoveryProvider | undefined => { + if (!input.embedder) return undefined; + const vector = makeVectorToolDiscoveryProvider({ + embedder: input.embedder, + store: input.store, + namespace: input.namespace, + }); + const lexicalProvider = input.lexicalStore + ? makeFtsLexicalProvider(input.lexicalStore, input.namespace) + : input.lexical; + return lexicalProvider + ? makeHybridToolDiscoveryProvider({ lexical: lexicalProvider, vector }) + : vector; +}; + +export const makeVectorToolSearchBackend = ( + options: VectorToolSearchBackendOptions, +): ToolSearchBackendFactory => { + const namespace = options.namespace ?? "default"; + const embedder = makeVectorEmbedder(options); + const chunker = options.chunker ?? makeFacetChunker(); + const provider = makeVectorProvider({ + namespace, + embedder, + store: options.store, + lexical: options.lexical, + lexicalStore: options.lexicalStore, + }); + + return { + namespace, + pluginStorage: { toolFingerprints, indexRuns, indexJobs, indexChunks }, + storage: (deps): VectorToolSearchBackendStorage => ({ + fingerprints: deps.pluginStorage.collection(toolFingerprints), + indexRuns: deps.pluginStorage.collection(indexRuns), + indexJobs: deps.pluginStorage.collection(indexJobs), + indexChunks: deps.pluginStorage.collection(indexChunks), + indexBlobs: deps.blobs, + // The tool catalog is an org-level artifact, so fingerprints are ALWAYS + // org-scoped. Scoping by the triggering principal would split the + // fingerprint store into disjoint partitions. + owner: "org" as const, + }), + build: ({ storage }) => { + const index = (executor: Executor): ToolSearchIndex.Service => + embedder && provider + ? makeToolSearchIndex({ + namespace, + executor, + embedder, + store: options.store, + chunker, + runs: storage.indexRuns, + jobs: storage.indexJobs, + chunks: storage.indexChunks, + fingerprints: storage.fingerprints, + blobs: storage.indexBlobs, + owner: storage.owner, + lexicalStore: options.lexicalStore, + }) + : unconfiguredIndex; + + return { + namespace, + provider, + index, + reindex: (executor) => + embedder + ? runToolSearchIndex({ + namespace, + executor, + embedder, + store: options.store, + chunker, + runs: storage.indexRuns, + jobs: storage.indexJobs, + chunks: storage.indexChunks, + fingerprints: storage.fingerprints, + blobs: storage.indexBlobs, + owner: storage.owner, + lexicalStore: options.lexicalStore, + runId: `manual-${Date.now()}`, + partitionCount: DEFAULT_IN_PROCESS_PARTITIONS, + }) + : notConfigured(), + sweep: (executor) => + sweepRemoved({ + namespace, + executor, + store: options.store, + fingerprints: storage.fingerprints, + blobs: storage.indexBlobs, + owner: storage.owner, + lexicalStore: options.lexicalStore, + }), + search: (_executor, input) => + provider + ? provider + .searchTools({ + executor: _executor, + query: input.query, + namespace: input.namespace, + limit: input.limit ?? DEFAULT_SEARCH_LIMIT, + offset: 0, + }) + .pipe( + Effect.map((page) => ({ + namespace, + query: input.query, + items: page.items, + })), + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Semantic search query failed.", cause }), + ), + ) + : notConfigured(), + status: () => + Effect.gen(function* () { + const indexed = yield* storage.fingerprints + .count() + .pipe( + Effect.mapError( + (cause) => + new SemanticSearchError({ message: "Failed to count indexed tools.", cause }), + ), + ); + const lexical = options.lexicalStore + ? yield* options.lexicalStore + .count(namespace) + .pipe(Effect.catch(() => Effect.succeed(null))) + : null; + return { namespace, indexed, lexical }; + }), + }; + }, + }; +}; + +export const ToolSearchBackend = { + vector: makeVectorToolSearchBackend, +} as const;