fix: respect caching: false on wrapAISDKModel

.changeset/wrap-aisdk-respect-caching-flag.md

@@ -0,0 +1,5 @@
+---
+"evalite": patch
+---
+
+Fix: `wrapAISDKModel(model, { caching: false })` is now actually respected. Previously the local `caching: false` flag was silently ignored as long as tracing was enabled — the cache was still being read from and written to. The wrapper now skips both cache fetch and store (in `wrapGenerate` and `wrapStream`) when `caching: false` is set.

packages/evalite-tests/tests/ai-sdk-caching.test.ts

@@ -94,3 +94,26 @@ it("Should let runEvalite cacheEnabled override config cacheEnabled", async () =
   // Should have no cache logs because runEvalite overrides config
   expect(cachelogs.length).toBe(0);
 });
+
+it("Should respect caching: false on wrapAISDKModel even when global cache is enabled", async () => {
+  await using fixture = await loadFixture("ai-sdk-caching-local-disabled");
+
+  // Two runs with the same input. Global cache is enabled (default), but each
+  // model is wrapped with `wrapAISDKModel(..., { caching: false })`, so neither
+  // run should fetch or store anything in the cache.
+  await fixture.run({
+    mode: "run-once-and-exit",
+    cacheDebug: true,
+    enableServer: true,
+  });
+
+  await fixture.run({
+    mode: "run-once-and-exit",
+    cacheDebug: true,
+    enableServer: true,
+  });
+
+  const allLogs = fixture.getOutput().split("\n");
+  const cachelogs = allLogs.filter((log) => log.includes("[CACHE]"));
+  expect(cachelogs.length).toBe(0);
+});

packages/evalite-tests/tests/fixtures/ai-sdk-caching-local-disabled/caching.eval.ts

@@ -0,0 +1,75 @@
+import { generateText } from "ai";
+import { MockLanguageModelV3 } from "ai/test";
+import { wrapAISDKModel } from "evalite/ai-sdk";
+import { evalite } from "evalite";
+
+const model = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 10,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 20, text: undefined, reasoning: undefined },
+    },
+    content: [{ type: "text", text: `Response for task` }],
+    warnings: [],
+  },
+});
+
+const scorerModel = new MockLanguageModelV3({
+  doGenerate: {
+    finishReason: { unified: "stop", raw: undefined },
+    usage: {
+      inputTokens: {
+        total: 5,
+        noCache: undefined,
+        cacheRead: undefined,
+        cacheWrite: undefined,
+      },
+      outputTokens: { total: 10, text: undefined, reasoning: undefined },
+    },
+    content: [{ type: "text", text: `1` }],
+    warnings: [],
+  },
+});
+
+const tracedModel = wrapAISDKModel(model, { caching: false });
+const tracedScorerModel = wrapAISDKModel(scorerModel, { caching: false });
+
+evalite("AI SDK Caching Local Disabled", {
+  data: () => {
+    return [
+      {
+        input: "test input 1",
+        expected: "expected output 1",
+      },
+      {
+        input: "test input 2",
+        expected: "expected output 2",
+      },
+    ];
+  },
+  task: async (input) => {
+    const result = await generateText({
+      model: tracedModel,
+      prompt: input,
+    });
+    return result.text;
+  },
+  scorers: [
+    {
+      name: "AI Scorer",
+      scorer: async ({ input, output, expected }) => {
+        const result = await generateText({
+          model: tracedScorerModel,
+          prompt: `Score this: ${output}`,
+        });
+        return { score: 1 };
+      },
+    },
+  ],
+});

packages/evalite/src/ai-sdk.ts

@@ -113,7 +113,7 @@ export const wrapAISDKModel = (
         const cacheContext = getCacheContext();
 
         // Try cache if enabled
-        if (cacheContext) {
+        if (cacheContext && enableCaching) {
           const keyHash = generateCacheKey({
             model: modelId,
             params: opts.params,
@@ -163,7 +163,7 @@ export const wrapAISDKModel = (
           const duration = performance.now() - start;
 
           // Store in cache if caching enabled
-          if (cacheContext) {
+          if (cacheContext && enableCaching) {
             const keyHash = generateCacheKey({
               model: modelId,
               params: opts.params,
@@ -237,7 +237,7 @@ export const wrapAISDKModel = (
         const reportTraceFromContext = reportTraceLocalStorage.getStore();
 
         // Try cache if enabled
-        if (cacheContext) {
+        if (cacheContext && enableCaching) {
           const keyHash = generateCacheKey({
             model: modelId,
             params: params,
@@ -318,7 +318,7 @@ export const wrapAISDKModel = (
               const duration = performance.now() - start;
 
               // Store in cache if enabled
-              if (cacheContext) {
+              if (cacheContext && enableCaching) {
                 const keyHash = generateCacheKey({
                   model: modelId,
                   params: params,