Fixed bugs and added tests

mattpocock · mattpocock · commit 9bf49cd9f867 · 2025-11-14T10:29:54.000Z
diff --git a/packages/evalite-tests/tests/ai-sdk-caching.test.ts b/packages/evalite-tests/tests/ai-sdk-caching.test.ts
@@ -16,14 +16,6 @@ it("Should cache AI SDK in the task and scorers", async () => {
     cacheDebug: true,
   });
 
-  const output = fixture.getOutput();
-
-  const storage = fixture.storage;
-
-  const runs = await storage.runs.getMany();
-
-  expect(runs).toHaveLength(2);
-
   const allLogs = fixture.getOutput().split("\n");
 
   const cachelogs = allLogs.filter((log) => log.includes("[CACHE]"));
@@ -32,3 +24,65 @@ it("Should cache AI SDK in the task and scorers", async () => {
   expect(cachelogs.some((log) => log.includes("Scorer cache HIT"))).toBe(true);
   expect(cachelogs.some((log) => log.includes("saved"))).toBe(true);
 });
+
+it("Should disable cache when cacheEnabled is false", async () => {
+  await using fixture = await loadFixture("ai-sdk-caching");
+
+  await fixture.run({
+    mode: "run-once-and-exit",
+    cacheDebug: true,
+    cacheEnabled: false,
+  });
+
+  await fixture.run({
+    mode: "run-once-and-exit",
+    cacheDebug: true,
+    cacheEnabled: false,
+  });
+
+  const allLogs = fixture.getOutput().split("\n");
+  const cachelogs = allLogs.filter((log) => log.includes("[CACHE]"));
+  expect(cachelogs.length).toBe(0);
+});
+
+it("Should respect cacheEnabled: false in config", async () => {
+  await using fixture = await loadFixture("ai-sdk-caching-config-disabled");
+
+  // First run
+  await fixture.run({
+    mode: "run-once-and-exit",
+    cacheDebug: true,
+  });
+
+  // Second run - should still not cache because config disables it
+  await fixture.run({
+    mode: "run-once-and-exit",
+    cacheDebug: true,
+  });
+
+  const allLogs = fixture.getOutput().split("\n");
+  const cachelogs = allLogs.filter((log) => log.includes("[CACHE]"));
+  expect(cachelogs.length).toBe(0);
+});
+
+it("Should let runEvalite cacheEnabled override config cacheEnabled", async () => {
+  await using fixture = await loadFixture("ai-sdk-caching-config-precedence");
+
+  // Config has cacheEnabled: true, but we override with false
+  await fixture.run({
+    mode: "run-once-and-exit",
+    cacheDebug: true,
+    cacheEnabled: false,
+  });
+
+  await fixture.run({
+    mode: "run-once-and-exit",
+    cacheDebug: true,
+    cacheEnabled: false,
+  });
+
+  const allLogs = fixture.getOutput().split("\n");
+  const cachelogs = allLogs.filter((log) => log.includes("[CACHE]"));
+  // Should have no cache logs because runEvalite overrides config
+  expect(cachelogs.length).toBe(0);
+});
diff --git a/packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-disabled/caching.eval.ts b/packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-disabled/caching.eval.ts
@@ -0,0 +1,63 @@
+import { generateText } from "ai";
+import { MockLanguageModelV2 } from "ai/test";
+import { wrapAISDKModel } from "evalite/ai-sdk";
+import { evalite } from "evalite";
+
+const model = new MockLanguageModelV2({
+  doGenerate: async (options) => ({
+    rawCall: { rawPrompt: null, rawSettings: {} },
+    finishReason: "stop",
+    usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+    content: [{ type: "text", text: `Response for task` }],
+    warnings: [],
+    providerMetadata: undefined,
+    request: undefined,
+    response: undefined,
+  }),
+});
+
+const scorerModel = new MockLanguageModelV2({
+  doGenerate: async (options) => ({
+    rawCall: { rawPrompt: null, rawSettings: {} },
+    finishReason: "stop",
+    usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 },
+    content: [{ type: "text", text: `1` }],
+    warnings: [],
+    providerMetadata: undefined,
+    request: undefined,
+    response: undefined,
+  }),
+});
+
+const tracedModel = wrapAISDKModel(model);
+const tracedScorerModel = wrapAISDKModel(scorerModel);
+
+evalite("AI SDK Caching Config Disabled", {
+  data: () => {
+    return [
+      {
+        input: "test input 1",
+        expected: "expected output 1",
+      },
+    ];
+  },
+  task: async (input) => {
+    const result = await generateText({
+      model: tracedModel,
+      prompt: input,
+    });
+    return result.text;
+  },
+  scorers: [
+    {
+      name: "AI Scorer",
+      scorer: async ({ input, output, expected }) => {
+        const result = await generateText({
+          model: tracedScorerModel,
+          prompt: `Score this: ${output}`,
+        });
+        return { score: 1 };
+      },
+    },
+  ],
+});
diff --git a/packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-disabled/evalite.config.ts b/packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-disabled/evalite.config.ts
@@ -0,0 +1,5 @@
+import { defineConfig } from "evalite/config";
+
+export default defineConfig({
+  cache: false,
+});
diff --git a/packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-precedence/caching.eval.ts b/packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-precedence/caching.eval.ts
@@ -0,0 +1,63 @@
+import { generateText } from "ai";
+import { MockLanguageModelV2 } from "ai/test";
+import { wrapAISDKModel } from "evalite/ai-sdk";
+import { evalite } from "evalite";
+
+const model = new MockLanguageModelV2({
+  doGenerate: async (options) => ({
+    rawCall: { rawPrompt: null, rawSettings: {} },
+    finishReason: "stop",
+    usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+    content: [{ type: "text", text: `Response for task` }],
+    warnings: [],
+    providerMetadata: undefined,
+    request: undefined,
+    response: undefined,
+  }),
+});
+
+const scorerModel = new MockLanguageModelV2({
+  doGenerate: async (options) => ({
+    rawCall: { rawPrompt: null, rawSettings: {} },
+    finishReason: "stop",
+    usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 },
+    content: [{ type: "text", text: `1` }],
+    warnings: [],
+    providerMetadata: undefined,
+    request: undefined,
+    response: undefined,
+  }),
+});
+
+const tracedModel = wrapAISDKModel(model);
+const tracedScorerModel = wrapAISDKModel(scorerModel);
+
+evalite("AI SDK Caching Config Precedence", {
+  data: () => {
+    return [
+      {
+        input: "test input 1",
+        expected: "expected output 1",
+      },
+    ];
+  },
+  task: async (input) => {
+    const result = await generateText({
+      model: tracedModel,
+      prompt: input,
+    });
+    return result.text;
+  },
+  scorers: [
+    {
+      name: "AI Scorer",
+      scorer: async ({ input, output, expected }) => {
+        const result = await generateText({
+          model: tracedScorerModel,
+          prompt: `Score this: ${output}`,
+        });
+        return { score: 1 };
+      },
+    },
+  ],
+});
diff --git a/packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-precedence/evalite.config.ts b/packages/evalite-tests/tests/fixtures/ai-sdk-caching-config-precedence/evalite.config.ts
@@ -0,0 +1,5 @@
+import { defineConfig } from "evalite/config";
+
+export default defineConfig({
+  cache: true,
+});
diff --git a/packages/evalite-tests/tests/test-utils.ts b/packages/evalite-tests/tests/test-utils.ts
@@ -61,7 +61,14 @@ export const loadFixture = async (
        * Set this to true if your test needs the server running (e.g., for cache functionality).
        */
       enableServer?: boolean;
+      /**
+       * Enable cache debug mode to log cache hits/misses.
+       */
       cacheDebug?: boolean;
+      /**
+       * Enable cache for AI SDK model outputs.
+       */
+      cacheEnabled?: boolean;
     }) => {
       const result = await runEvalite({
         ...opts,
@@ -70,6 +77,7 @@ export const loadFixture = async (
         testOutputWritable: captured.writable,
         disableServer: !opts.enableServer,
         cacheDebug: opts.cacheDebug ?? false,
+        cacheEnabled: opts.cacheEnabled,
       });
       vitestInstance = result.vitest;
       return vitestInstance;
diff --git a/packages/evalite/src/evalite.ts b/packages/evalite/src/evalite.ts
@@ -61,6 +61,7 @@ const runTask = async <TInput, TOutput, TExpected, TVariant = undefined>(
     traces: Evalite.Trace[];
     cacheContext: CacheContextConfig;
     cacheDebug: boolean;
+    cacheEnabled: boolean;
   } & Omit<Evalite.RunnerOpts<TInput, TOutput, TExpected, TVariant>, "data">
 ) => {
   const start = performance.now();
@@ -81,6 +82,9 @@ const runTask = async <TInput, TOutput, TExpected, TVariant = undefined>(
             {
               ...opts.cacheContext,
               reportCacheHit: (hit) => {
+                if (!opts.cacheEnabled) {
+                  return;
+                }
                 scorerCacheHits.push(hit);
                 if (opts.cacheDebug) {
                   console.log(
@@ -356,10 +360,14 @@ function registerEvalite<TInput, TOutput, TExpected>(
         };
 
         const cacheDebug = inject("cacheDebug");
+        const cacheEnabled = inject("cacheEnabled");
 
         cacheContextLocalStorage.enterWith({
           ...cacheContext,
           reportCacheHit: (hit) => {
+            if (!cacheEnabled) {
+              return;
+            }
             taskCacheHits.push(hit);
             if (cacheDebug) {
               console.log(
@@ -390,6 +398,7 @@ function registerEvalite<TInput, TOutput, TExpected>(
             traces,
             cacheContext,
             cacheDebug,
+            cacheEnabled,
           });
 
           const [outputWithFiles, tracesWithFiles, renderedColumns] =
diff --git a/packages/evalite/src/run-evalite.ts b/packages/evalite/src/run-evalite.ts
@@ -31,6 +31,10 @@ declare module "vitest" {
      * Whether to log cache operations to the console.
      */
     cacheDebug: boolean;
+    /**
+     * Whether to enable cache for AI SDK model outputs.
+     */
+    cacheEnabled: boolean;
   }
 }
 
@@ -246,15 +250,7 @@ export const runEvalite = async (opts: {
   const maxConcurrency = config?.maxConcurrency;
 
   // Determine cache enabled: opts > config > default (true)
-  let cacheEnabled = true;
-  if (opts.cacheEnabled !== undefined) {
-    cacheEnabled = opts.cacheEnabled;
-  } else if (config?.cache !== undefined) {
-    cacheEnabled =
-      typeof config.cache === "boolean"
-        ? config.cache
-        : (config.cache.enabled ?? true);
-  }
+  const cacheEnabled = opts.cacheEnabled ?? config?.cache ?? true;
 
   // Merge setupFiles:
   // 1. Always include env-setup-file first to load .env files
@@ -355,6 +351,7 @@ export const runEvalite = async (opts: {
   vitest.provide("trialCount", config?.trialCount);
   vitest.provide("serverPort", actualServerPort);
   vitest.provide("cacheDebug", opts.cacheDebug ?? false);
+  vitest.provide("cacheEnabled", cacheEnabled);
 
   await vitest.start(filters);
 
diff --git a/packages/evalite/src/types.ts b/packages/evalite/src/types.ts
@@ -103,29 +103,15 @@ export declare namespace Evalite {
 
     /**
      * Cache configuration for AI SDK model outputs
-     * @default { enabled: true, ttlDays: 30 }
+     * @default true
      * @example
      * ```ts
      * export default defineConfig({
      *   cache: false // Disable cache entirely
      * })
      * ```
-     * @example
-     * ```ts
-     * export default defineConfig({
-     *   cache: {
-     *     enabled: true,
-     *     ttlDays: 7 // Cache for 7 days
-     *   }
-     * })
-     * ```
      */
-    cache?:
-      | boolean
-      | {
-          enabled?: boolean;
-          ttlDays?: number;
-        };
+    cache?: boolean;
 
     /**
      * Pass-through Vite/Vitest configuration options.
