diff --git a/.changeset/wrap-aisdk-respect-caching-flag.md b/.changeset/wrap-aisdk-respect-caching-flag.md new file mode 100644 index 00000000..e3684eb4 --- /dev/null +++ b/.changeset/wrap-aisdk-respect-caching-flag.md @@ -0,0 +1,5 @@ +--- +"evalite": patch +--- + +Fix: `wrapAISDKModel(model, { caching: false })` is now actually respected. Previously the local `caching: false` flag was silently ignored as long as tracing was enabled — the cache was still being read from and written to. The wrapper now skips both cache fetch and store (in `wrapGenerate` and `wrapStream`) when `caching: false` is set. diff --git a/packages/evalite-tests/tests/ai-sdk-caching.test.ts b/packages/evalite-tests/tests/ai-sdk-caching.test.ts index e5083668..68e370c6 100644 --- a/packages/evalite-tests/tests/ai-sdk-caching.test.ts +++ b/packages/evalite-tests/tests/ai-sdk-caching.test.ts @@ -94,3 +94,26 @@ it("Should let runEvalite cacheEnabled override config cacheEnabled", async () = // Should have no cache logs because runEvalite overrides config expect(cachelogs.length).toBe(0); }); + +it("Should respect caching: false on wrapAISDKModel even when global cache is enabled", async () => { + await using fixture = await loadFixture("ai-sdk-caching-local-disabled"); + + // Two runs with the same input. Global cache is enabled (default), but each + // model is wrapped with `wrapAISDKModel(..., { caching: false })`, so neither + // run should fetch or store anything in the cache. + await fixture.run({ + mode: "run-once-and-exit", + cacheDebug: true, + enableServer: true, + }); + + await fixture.run({ + mode: "run-once-and-exit", + cacheDebug: true, + enableServer: true, + }); + + const allLogs = fixture.getOutput().split("\n"); + const cachelogs = allLogs.filter((log) => log.includes("[CACHE]")); + expect(cachelogs.length).toBe(0); +}); diff --git a/packages/evalite-tests/tests/fixtures/ai-sdk-caching-local-disabled/caching.eval.ts b/packages/evalite-tests/tests/fixtures/ai-sdk-caching-local-disabled/caching.eval.ts new file mode 100644 index 00000000..40a2daf1 --- /dev/null +++ b/packages/evalite-tests/tests/fixtures/ai-sdk-caching-local-disabled/caching.eval.ts @@ -0,0 +1,75 @@ +import { generateText } from "ai"; +import { MockLanguageModelV3 } from "ai/test"; +import { wrapAISDKModel } from "evalite/ai-sdk"; +import { evalite } from "evalite"; + +const model = new MockLanguageModelV3({ + doGenerate: { + finishReason: { unified: "stop", raw: undefined }, + usage: { + inputTokens: { + total: 10, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: 20, text: undefined, reasoning: undefined }, + }, + content: [{ type: "text", text: `Response for task` }], + warnings: [], + }, +}); + +const scorerModel = new MockLanguageModelV3({ + doGenerate: { + finishReason: { unified: "stop", raw: undefined }, + usage: { + inputTokens: { + total: 5, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: 10, text: undefined, reasoning: undefined }, + }, + content: [{ type: "text", text: `1` }], + warnings: [], + }, +}); + +const tracedModel = wrapAISDKModel(model, { caching: false }); +const tracedScorerModel = wrapAISDKModel(scorerModel, { caching: false }); + +evalite("AI SDK Caching Local Disabled", { + data: () => { + return [ + { + input: "test input 1", + expected: "expected output 1", + }, + { + input: "test input 2", + expected: "expected output 2", + }, + ]; + }, + task: async (input) => { + const result = await generateText({ + model: tracedModel, + prompt: input, + }); + return result.text; + }, + scorers: [ + { + name: "AI Scorer", + scorer: async ({ input, output, expected }) => { + const result = await generateText({ + model: tracedScorerModel, + prompt: `Score this: ${output}`, + }); + return { score: 1 }; + }, + }, + ], +}); diff --git a/packages/evalite/src/ai-sdk.ts b/packages/evalite/src/ai-sdk.ts index 04ec9f4d..ca7f3fb3 100644 --- a/packages/evalite/src/ai-sdk.ts +++ b/packages/evalite/src/ai-sdk.ts @@ -113,7 +113,7 @@ export const wrapAISDKModel = ( const cacheContext = getCacheContext(); // Try cache if enabled - if (cacheContext) { + if (cacheContext && enableCaching) { const keyHash = generateCacheKey({ model: modelId, params: opts.params, @@ -163,7 +163,7 @@ export const wrapAISDKModel = ( const duration = performance.now() - start; // Store in cache if caching enabled - if (cacheContext) { + if (cacheContext && enableCaching) { const keyHash = generateCacheKey({ model: modelId, params: opts.params, @@ -237,7 +237,7 @@ export const wrapAISDKModel = ( const reportTraceFromContext = reportTraceLocalStorage.getStore(); // Try cache if enabled - if (cacheContext) { + if (cacheContext && enableCaching) { const keyHash = generateCacheKey({ model: modelId, params: params, @@ -318,7 +318,7 @@ export const wrapAISDKModel = ( const duration = performance.now() - start; // Store in cache if enabled - if (cacheContext) { + if (cacheContext && enableCaching) { const keyHash = generateCacheKey({ model: modelId, params: params,