Got it half working

Author: mattpocock

apps/evalite-ui/app/routes/suite.$name.tsx

@@ -71,6 +71,7 @@ type EvalTableRowProps = {
   hasScores: boolean;
   prevSuite: Evalite.SDK.GetSuiteByNameResult["prevSuite"];
   cacheHitCount: number;
+  cacheHitsByScorer: Record<string, number>;
   trialConfig?: {
     isFirstTrial: boolean;
     rowSpan: number;
@@ -111,6 +112,7 @@ function EvalTableRow({
   hasScores,
   prevSuite: prevEvaluation,
   cacheHitCount,
+  cacheHitsByScorer,
   trialConfig,
 }: EvalTableRowProps) {
   const Wrapper = useMemo(
@@ -202,10 +204,24 @@ function EvalTableRow({
         const scoreInPreviousEvaluation = prevEvaluation?.evals
           .find((r) => r.input === _eval.input)
           ?.scores.find((s) => s.name === scorer.name);
+        const scorerCacheHitCount = cacheHitsByScorer[scorer.name] ?? 0;
         return (
           <TableCell key={scorer.id} className={cn(index === 0 && "border-l")}>
             <Wrapper>
               <div className="flex items-center gap-2">
+                {scorerCacheHitCount > 0 && (
+                  <Tooltip>
+                    <TooltipTrigger asChild>
+                      <Zap className="size-3 text-accent-foreground" />
+                    </TooltipTrigger>
+                    <TooltipContent>
+                      {scorerCacheHitCount} LLM{" "}
+                      {scorerCacheHitCount === 1
+                        ? "call was cached"
+                        : "calls were cached"}
+                    </TooltipContent>
+                  </Tooltip>
+                )}
                 <Score
                   hasScores={hasScores}
                   score={scorer.score}
@@ -541,6 +557,8 @@ function SuiteComponent() {
                           evaluationWithoutLayoutShift!.evals.indexOf(_eval);
                         const cacheHitCount =
                           serverState.cacheHitsByEval[_eval.id] ?? 0;
+                        const cacheHitsByScorer =
+                          serverState.cacheHitsByScorer[_eval.id] ?? {};
                         return (
                           <EvalTableRow
                             key={`${JSON.stringify(_eval.input)}-${_eval.trial_index}`}
@@ -553,6 +571,7 @@ function SuiteComponent() {
                             hasScores={hasScores}
                             prevSuite={prevSuite}
                             cacheHitCount={cacheHitCount}
+                            cacheHitsByScorer={cacheHitsByScorer}
                             trialConfig={{
                               isFirstTrial: trialIndex === 0,
                               rowSpan: group.evals.length,
@@ -566,6 +585,8 @@ function SuiteComponent() {
                     evaluationWithoutLayoutShift.evals.map((_eval, index) => {
                       const cacheHitCount =
                         serverState.cacheHitsByEval[_eval.id] ?? 0;
+                      const cacheHitsByScorer =
+                        serverState.cacheHitsByScorer[_eval.id] ?? {};
                       return (
                         <EvalTableRow
                           key={JSON.stringify(_eval.input)}
@@ -578,6 +599,7 @@ function SuiteComponent() {
                           hasScores={hasScores}
                           prevSuite={prevSuite}
                           cacheHitCount={cacheHitCount}
+                          cacheHitsByScorer={cacheHitsByScorer}
                         />
                       );
                     })}

packages/evalite/src/ai-sdk.ts

@@ -5,7 +5,7 @@ import type {
 } from "@ai-sdk/provider";
 import { wrapLanguageModel } from "ai";
 import { reportTrace, shouldReportTrace } from "./traces.js";
-import { getCacheContext, generateCacheKey, reportCacheHit } from "./cache.js";
+import { getCacheContext, generateCacheKey } from "./cache.js";
 
 const handlePromptContent = (
   content: LanguageModelV2CallOptions["prompt"][number]["content"][number]
@@ -98,6 +98,7 @@ export const wrapAISDKModel = (
   const context = getCacheContext();
   const cachingAvailable =
     enableCaching && (context?.cacheEnabled ?? true) && context?.serverPort;
+
   const tracingAvailable = enableTracing && shouldReportTrace();
 
   // If neither is enabled/available, return original model
@@ -133,7 +134,7 @@ export const wrapAISDKModel = (
                 duration: number;
               };
               if (cached?.value) {
-                reportCacheHit({
+                context.reportCacheHit({
                   keyHash,
                   hit: true,
                   savedDuration: cached.duration,
@@ -182,7 +183,7 @@ export const wrapAISDKModel = (
               console.warn("Cache write failed:", error);
             }
 
-            reportCacheHit({ keyHash, hit: false, savedDuration: 0 });
+            context.reportCacheHit({ keyHash, hit: false, savedDuration: 0 });
           }
         }
 
@@ -250,7 +251,7 @@ export const wrapAISDKModel = (
                 duration: number;
               };
               if (cached?.value) {
-                reportCacheHit({
+                context.reportCacheHit({
                   keyHash,
                   hit: true,
                   savedDuration: cached.duration,
@@ -343,7 +344,11 @@ export const wrapAISDKModel = (
                   console.warn("Cache write failed:", error);
                 }
 
-                reportCacheHit({ keyHash, hit: false, savedDuration: 0 });
+                context.reportCacheHit({
+                  keyHash,
+                  hit: false,
+                  savedDuration: 0,
+                });
               }
 
               // Report trace if enabled

packages/evalite/src/cache.ts

@@ -1,12 +1,22 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import { createHash } from "node:crypto";
 
-export const cacheContextLocalStorage = new AsyncLocalStorage<{
+export type CacheContextConfig = {
   trialCount: number | undefined;
   evalName: string;
   serverPort: number;
   cacheEnabled: boolean;
-}>();
+};
+
+export interface CacheContext extends CacheContextConfig {
+  reportCacheHit: (hit: {
+    keyHash: string;
+    hit: boolean;
+    savedDuration: number;
+  }) => void;
+}
+
+export const cacheContextLocalStorage = new AsyncLocalStorage<CacheContext>();
 
 export const getCacheContext = () => {
   return cacheContextLocalStorage.getStore();
@@ -30,19 +40,3 @@ export const generateCacheKey = (params: {
 
   return createHash("sha256").update(JSON.stringify(cacheObject)).digest("hex");
 };
-
-export const reportCacheHitLocalStorage = new AsyncLocalStorage<
-  (hit: { keyHash: string; hit: boolean; savedDuration: number }) => void
->();
-
-export const reportCacheHit = (hit: {
-  keyHash: string;
-  hit: boolean;
-  savedDuration: number;
-}): void => {
-  const _reportCacheHit = reportCacheHitLocalStorage.getStore();
-
-  if (_reportCacheHit) {
-    _reportCacheHit(hit);
-  }
-};

packages/evalite/src/evalite.ts

@@ -8,10 +8,7 @@ import type { Evalite } from "./types.js";
 import { FILES_LOCATION } from "./backend-only-constants.js";
 import { createScorer } from "./index.js";
 import { serializeAnnotation } from "./reporter/events.js";
-import {
-  cacheContextLocalStorage,
-  reportCacheHitLocalStorage,
-} from "./cache.js";
+import { cacheContextLocalStorage, type CacheContextConfig } from "./cache.js";
 
 const makeSerializable = (obj: unknown): unknown => {
   try {
@@ -62,27 +59,53 @@ const runTask = async <TInput, TOutput, TExpected, TVariant = undefined>(
     expected: TExpected | undefined;
     variant: TVariant;
     traces: Evalite.Trace[];
+    cacheContext: CacheContextConfig;
   } & Omit<Evalite.RunnerOpts<TInput, TOutput, TExpected, TVariant>, "data">
 ) => {
   const start = performance.now();
   const output = await executeTask(opts.task, opts.input, opts.variant);
   const duration = Math.round(performance.now() - start);
 
   const scores = await Promise.all(
-    (opts.scorers || []).map(async (scorerOrOpts) => {
-      if (typeof scorerOrOpts === "function") {
-        return scorerOrOpts({
-          input: opts.input,
-          output,
-          expected: opts.expected as TExpected,
-        });
-      } else {
-        return createScorer(scorerOrOpts)({
-          input: opts.input,
-          output,
-          expected: opts.expected as TExpected,
-        });
-      }
+    (opts.scorers || []).map(async (scorerOrOpts, index) => {
+      // Isolate scorer traces - LLM calls in scorers still get traced
+      // but traces are discarded (not collected in parent eval)
+      return reportTraceLocalStorage.run(
+        () => {
+          // no-op: discard traces
+        },
+        () => {
+          const scorerCacheHits: Array<Evalite.CacheHit> = [];
+          return cacheContextLocalStorage.run(
+            {
+              ...opts.cacheContext,
+              reportCacheHit: (hit) => {
+                scorerCacheHits.push(hit);
+              },
+            },
+            async (): Promise<Evalite.ScoreWithCacheHits> => {
+              const score =
+                typeof scorerOrOpts === "function"
+                  ? await scorerOrOpts({
+                      input: opts.input,
+                      output,
+                      expected: opts.expected as TExpected,
+                    })
+                  : await createScorer(scorerOrOpts)({
+                      input: opts.input,
+                      output,
+                      expected: opts.expected as TExpected,
+                    });
+
+              // Attach cache hits to score if there were any
+              return {
+                ...score,
+                cacheHits: scorerCacheHits,
+              };
+            }
+          );
+        }
+      );
     })
   );
 
@@ -224,7 +247,7 @@ function registerEvalite<TInput, TOutput, TExpected>(
               output: datasetResult.error,
               scores: [],
               traces: [],
-              cacheHits: [],
+              taskCacheHits: [],
               renderedColumns: [],
             },
           })
@@ -318,18 +341,18 @@ function registerEvalite<TInput, TOutput, TExpected>(
         const traces: Evalite.Trace[] = [];
         reportTraceLocalStorage.enterWith((trace) => traces.push(trace));
 
-        const cacheHits: Array<{
-          keyHash: string;
-          hit: boolean;
-          savedDuration: number;
-        }> = [];
-        reportCacheHitLocalStorage.enterWith((hit) => cacheHits.push(hit));
+        const taskCacheHits: Array<Evalite.CacheHit> = [];
 
-        cacheContextLocalStorage.enterWith({
+        const cacheContext: CacheContextConfig = {
           trialCount: inject("trialCount"),
           evalName: evalName,
           serverPort: inject("serverPort"),
           cacheEnabled: inject("cacheEnabled"),
+        };
+
+        cacheContextLocalStorage.enterWith({
+          ...cacheContext,
+          reportCacheHit: (hit) => taskCacheHits.push(hit),
         });
 
         const [inputForMeta, expectedForMeta] = await Promise.all([
@@ -351,6 +374,7 @@ function registerEvalite<TInput, TOutput, TExpected>(
             task: opts.task,
             columns: opts.columns,
             traces,
+            cacheContext,
           });
 
           const [outputWithFiles, tracesWithFiles, renderedColumns] =
@@ -379,7 +403,7 @@ function registerEvalite<TInput, TOutput, TExpected>(
                 output: serializableOutput,
                 scores,
                 traces: tracesWithFiles,
-                cacheHits,
+                taskCacheHits: taskCacheHits,
                 status: "success",
                 renderedColumns,
                 variantName: vitestOpts.variantName,
@@ -415,7 +439,7 @@ function registerEvalite<TInput, TOutput, TExpected>(
                 output: serializedError,
                 scores: [],
                 traces: await handleFilesInTraces(rootDir, traces),
-                cacheHits,
+                taskCacheHits: taskCacheHits,
                 status: "fail",
                 renderedColumns: [],
                 variantName: vitestOpts.variantName,

packages/evalite/src/export-static.ts

@@ -275,6 +275,7 @@ export const exportStaticUI = async (
   const serverState: Evalite.ServerState = {
     type: "idle",
     cacheHitsByEval: {},
+    cacheHitsByScorer: {},
   };
   await fs.writeFile(
     path.join(dataDir, "server-state.json"),

packages/evalite/src/reporter.ts

@@ -368,7 +368,7 @@ export default class EvaliteReporter implements Reporter {
           output: null,
           scores: [],
           traces: [],
-          cacheHits: [],
+          taskCacheHits: [],
           status: "fail",
           renderedColumns: [],
           variantName: data.initialEval.variantName,

packages/evalite/src/reporter/EvaliteRunner.ts

@@ -10,7 +10,11 @@ export interface EvaliteRunnerOptions {
 
 export class EvaliteRunner {
   private opts: EvaliteRunnerOptions;
-  private state: Evalite.ServerState = { type: "idle", cacheHitsByEval: {} };
+  private state: Evalite.ServerState = {
+    type: "idle",
+    cacheHitsByEval: {},
+    cacheHitsByScorer: {},
+  };
   private didLastRunFailThreshold: "yes" | "no" | "unknown" = "unknown";
   private collectedResults: Map<string, Evalite.Eval> = new Map();
   private eventQueue: Promise<void> = Promise.resolve();
@@ -121,6 +125,7 @@ export class EvaliteRunner {
             this.updateState({
               type: "idle",
               cacheHitsByEval: this.state.cacheHitsByEval,
+              cacheHitsByScorer: this.state.cacheHitsByScorer,
             });
             break;
           case "EVAL_STARTED":
@@ -271,13 +276,25 @@ export class EvaliteRunner {
               }
 
               // Count cache hits for this eval
-              const cacheHitCount = event.eval.cacheHits.filter(
+              const cacheHitCount = event.eval.taskCacheHits.filter(
                 (hit) => hit.hit
               ).length;
               if (cacheHitCount > 0) {
                 this.state.cacheHitsByEval[evalId] = cacheHitCount;
               }
 
+              // Count cache hits per scorer
+              if (!this.state.cacheHitsByScorer[evalId]) {
+                this.state.cacheHitsByScorer[evalId] = {};
+              }
+              for (const score of event.eval.scores) {
+                const scorerCacheHits = score.cacheHits;
+                if (scorerCacheHits && scorerCacheHits.length > 0) {
+                  this.state.cacheHitsByScorer[evalId]![score.name] =
+                    scorerCacheHits.length;
+                }
+              }
+
               const allEvals = await this.opts.storage.evals.getMany({
                 suiteIds: [suite.id],
               });
@@ -327,6 +344,7 @@ export class EvaliteRunner {
             // Clear collected results for new run
             this.collectedResults.clear();
             this.state.cacheHitsByEval = {};
+            this.state.cacheHitsByScorer = {};
 
             this.updateState({
               filepaths: event.filepaths,
@@ -336,6 +354,7 @@ export class EvaliteRunner {
               suiteNamesRunning: [],
               evalIdsRunning: [],
               cacheHitsByEval: this.state.cacheHitsByEval,
+              cacheHitsByScorer: this.state.cacheHitsByScorer,
             });
             break;
         }