diff --git a/packages/plugins/semantic-search/src/sdk/collections.ts b/packages/plugins/semantic-search/src/sdk/collections.ts index d17064e4e..722fe09bb 100644 --- a/packages/plugins/semantic-search/src/sdk/collections.ts +++ b/packages/plugins/semantic-search/src/sdk/collections.ts @@ -70,6 +70,7 @@ export const IndexJob = Schema.Struct({ name: Schema.String, integration: Schema.String, description: Schema.String, + sourceRevision: Schema.optional(Schema.String), status: IndexJobStatus, fingerprint: Schema.optional(Schema.String), oldFingerprint: Schema.optional(Schema.String), diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts index 1c1b77bb2..111b1a147 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.test.ts @@ -67,6 +67,7 @@ const makeMemoryCache = (): Executor["cache"] => { const manifestForTool = ( tool: Tool, fingerprint = `fingerprint:${String(tool.address)}`, + sourceRevision?: string, ): ToolSchemaManifest => ({ address: tool.address, path: String(tool.address).replace(/^tools\./, ""), @@ -82,6 +83,7 @@ const manifestForTool = ( definitionSetHash: `definitions:${String(tool.address)}`, indexFingerprint: fingerprint, fingerprintVersion: "tool-schema-manifest/v1", + ...(sourceRevision === undefined ? {} : { sourceRevision }), }); const makeExecutor = ( @@ -327,6 +329,7 @@ describe("ToolSearchIndex", () => { } expect(scanned).toMatchObject({ processed: 1, changed: 1, skipped: 0 }); + expect([...jobs.data.values()][0]?.sourceRevision).toBeUndefined(); expect(chunked.processed).toBe(1); expect(chunked.chunks).toBeGreaterThan(0); expect(embedded).toMatchObject({ processed: chunked.chunks, chunks: chunked.chunks }); @@ -350,6 +353,84 @@ describe("ToolSearchIndex", () => { }), ); + it.effect("copies manifest source revisions into scan jobs", () => + Effect.gen(function* () { + const tool: Tool = { + address: "tools.github.repos.get" as never, + name: "repos.get" as never, + integration: "github" as never, + description: "Get a repository", + owner, + connection: "default" as never, + pluginId: "test", + }; + const executor: Pick = { + tools: { + list: () => Effect.succeed([tool]), + manifest: () => Effect.succeed([manifestForTool(tool, "fp-source", "spec-hash-v1")]), + schema: () => Effect.succeed(null), + }, + cache: makeMemoryCache(), + }; + const base = { + namespace, + executor: executor as Executor, + runs: makeCollection(indexRuns.name), + jobs: makeCollection(indexJobs.name), + chunks: makeCollection(indexChunks.name), + fingerprints: makeCollection(toolFingerprints.name), + blobs: makeBlobs(), + owner, + }; + + yield* create({ ...base, runId: "run-source-revision", partitionCount: 1 }); + const scanned = yield* scan({ + ...base, + runId: "run-source-revision", + partition: 0, + limit: 10, + }); + + expect(scanned).toMatchObject({ processed: 1, changed: 1, skipped: 0 }); + expect([...base.jobs.data.values()][0]).toMatchObject({ + path: "github.repos.get", + fingerprint: "fp-source", + sourceRevision: "spec-hash-v1", + }); + + const skippedBase = { + ...base, + jobs: makeCollection(indexJobs.name), + fingerprints: makeCollection(toolFingerprints.name), + }; + yield* skippedBase.fingerprints.put({ + owner, + key: "github.repos.get", + data: { + path: "github.repos.get", + integration: "github", + fingerprint: "fp-source", + chunkIds: [], + }, + }); + + yield* create({ ...skippedBase, runId: "run-source-revision-skipped", partitionCount: 1 }); + const skipped = yield* scan({ + ...skippedBase, + runId: "run-source-revision-skipped", + partition: 0, + limit: 10, + }); + + expect(skipped).toMatchObject({ processed: 1, changed: 0, skipped: 1 }); + expect([...skippedBase.jobs.data.values()][0]).toMatchObject({ + path: "github.repos.get", + fingerprint: "fp-source", + sourceRevision: "spec-hash-v1", + }); + }), + ); + it.effect("marks pending path work failed and reports progress timestamps", () => Effect.gen(function* () { const counters = { raw: 0, codegen: 0 }; diff --git a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts index 2a2b539fa..07acafda0 100644 --- a/packages/plugins/semantic-search/src/sdk/tool-search-index.ts +++ b/packages/plugins/semantic-search/src/sdk/tool-search-index.ts @@ -1092,6 +1092,9 @@ export const scan = ( name: manifest.name, integration: manifest.integration, description: manifest.description, + ...(manifest.sourceRevision === undefined + ? {} + : { sourceRevision: manifest.sourceRevision }), status: "skipped", fingerprint: manifest.indexFingerprint, oldFingerprint: storedRow.fingerprint, @@ -1110,6 +1113,9 @@ export const scan = ( name: manifest.name, integration: manifest.integration, description: manifest.description, + ...(manifest.sourceRevision === undefined + ? {} + : { sourceRevision: manifest.sourceRevision }), status: "pendingChunk", fingerprint: manifest.indexFingerprint, oldFingerprint: storedRow?.fingerprint,