Showed cache hits in the UI

mattpocock · mattpocock · commit c1a2208ec575 · 2025-11-10T20:57:37.000Z
diff --git a/apps/evalite-ui/app/constants.ts b/apps/evalite-ui/app/constants.ts
@@ -0,0 +1,3 @@
+export const BASE_URL = import.meta.env.DEV
+  ? "http://localhost:3006"
+  : window.location.origin;
diff --git a/apps/evalite-ui/app/data/use-subscribe-to-socket.ts b/apps/evalite-ui/app/data/use-subscribe-to-socket.ts
@@ -3,6 +3,7 @@ import type { QueryClient } from "@tanstack/react-query";
 import { getServerStateQueryOptions } from "./queries";
 import type { Evalite } from "evalite/types";
 import { isStaticMode } from "~/sdk";
+import { BASE_URL } from "~/constants";
 
 export const useSubscribeToSocket = (queryClient: QueryClient) => {
   useEffect(() => {
@@ -11,9 +12,7 @@ export const useSubscribeToSocket = (queryClient: QueryClient) => {
       return;
     }
 
-    const socket = new WebSocket(
-      `${window.location.origin}/api/socket`
-    );
+    const socket = new WebSocket(`${BASE_URL}/api/socket`);
 
     socket.onmessage = async (event) => {
       const newState: Evalite.ServerState = JSON.parse(event.data);
diff --git a/apps/evalite-ui/app/routes/suite.$name.tsx b/apps/evalite-ui/app/routes/suite.$name.tsx
@@ -2,7 +2,7 @@ import { createFileRoute } from "@tanstack/react-router";
 import { z } from "zod";
 import { zodValidator } from "@tanstack/zod-adapter";
 import { Link, Outlet, useMatches } from "@tanstack/react-router";
-import { XCircleIcon } from "lucide-react";
+import { XCircleIcon, Zap } from "lucide-react";
 import type * as React from "react";
 
 import { DisplayInput } from "~/components/display-input";
@@ -19,6 +19,11 @@ import {
   TableHeader,
   TableRow,
 } from "~/components/ui/table";
+import {
+  Tooltip,
+  TooltipTrigger,
+  TooltipContent,
+} from "~/components/ui/tooltip";
 import { cn } from "~/lib/utils";
 import { formatTime, isArrayOfRenderedColumns } from "~/utils";
 import { useServerStateUtils } from "~/hooks/use-server-state-utils";
@@ -65,6 +70,7 @@ type EvalTableRowProps = {
   isRunningEval: boolean;
   hasScores: boolean;
   prevSuite: Evalite.SDK.GetSuiteByNameResult["prevSuite"];
+  cacheHitCount: number;
   trialConfig?: {
     isFirstTrial: boolean;
     rowSpan: number;
@@ -104,17 +110,31 @@ function EvalTableRow({
   isRunningEval,
   hasScores,
   prevSuite: prevEvaluation,
+  cacheHitCount,
   trialConfig,
 }: EvalTableRowProps) {
   const Wrapper = useMemo(
     () => makeWrapper({ evalIndex, timestamp, name }),
     [evalIndex, timestamp, name]
   );
+
   return (
     <TableRow className={cn("has-[.active]:bg-foreground/20!")}>
+      {cacheHitCount > 0 && (
+        <TableCell className="pt-4 pl-4">
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <Zap className="size-4 text-accent-foreground" />
+            </TooltipTrigger>
+            <TooltipContent>
+              {cacheHitCount} cache {cacheHitCount === 1 ? "hit" : "hits"}
+            </TooltipContent>
+          </Tooltip>
+        </TableCell>
+      )}
       {isArrayOfRenderedColumns(_eval.rendered_columns) ? (
         <>
-          {_eval.rendered_columns.map((column) => (
+          {_eval.rendered_columns.map((column, index) => (
             <TableCell>
               <DisplayInput
                 className={cn(
@@ -185,15 +205,17 @@ function EvalTableRow({
         return (
           <TableCell key={scorer.id} className={cn(index === 0 && "border-l")}>
             <Wrapper>
-              <Score
-                hasScores={hasScores}
-                score={scorer.score}
-                state={getScoreState({
-                  score: scorer.score,
-                  prevScore: scoreInPreviousEvaluation?.score,
-                  status: _eval.status,
-                })}
-              />
+              <div className="flex items-center gap-2">
+                <Score
+                  hasScores={hasScores}
+                  score={scorer.score}
+                  state={getScoreState({
+                    score: scorer.score,
+                    prevScore: scoreInPreviousEvaluation?.score,
+                    status: _eval.status,
+                  })}
+                />
+              </div>
             </Wrapper>
           </TableCell>
         );
@@ -326,6 +348,10 @@ function SuiteComponent() {
   const hasScores =
     possiblyRunningSuite.evals.some((r) => r.scores.length > 0) ?? true;
 
+  const doAnyEvalsHaveCacheHits = Object.values(
+    serverState.cacheHitsByEval
+  ).some((hits) => hits > 0);
+
   return (
     <>
       <title>{`${name} | Evalite`}</title>
@@ -474,6 +500,7 @@ function SuiteComponent() {
             <Table>
               <TableHeader>
                 <TableRow>
+                  {doAnyEvalsHaveCacheHits && <TableHead></TableHead>}
                   {isArrayOfRenderedColumns(
                     evaluationWithoutLayoutShift.evals[0]?.rendered_columns
                   ) ? (
@@ -512,6 +539,8 @@ function SuiteComponent() {
                       group.evals.map((_eval, trialIndex) => {
                         const evalIndex =
                           evaluationWithoutLayoutShift!.evals.indexOf(_eval);
+                        const cacheHitCount =
+                          serverState.cacheHitsByEval[_eval.id] ?? 0;
                         return (
                           <EvalTableRow
                             key={`${JSON.stringify(_eval.input)}-${_eval.trial_index}`}
@@ -523,6 +552,7 @@ function SuiteComponent() {
                             isRunningEval={isRunningEval}
                             hasScores={hasScores}
                             prevSuite={prevSuite}
+                            cacheHitCount={cacheHitCount}
                             trialConfig={{
                               isFirstTrial: trialIndex === 0,
                               rowSpan: group.evals.length,
@@ -533,19 +563,24 @@ function SuiteComponent() {
                       })
                     )
                   : // Original rendering for non-trial results
-                    evaluationWithoutLayoutShift.evals.map((_eval, index) => (
-                      <EvalTableRow
-                        key={JSON.stringify(_eval.input)}
-                        eval={_eval}
-                        evalIndex={index}
-                        name={name}
-                        timestamp={timestamp}
-                        showExpectedColumn={showExpectedColumn}
-                        isRunningEval={isRunningEval}
-                        hasScores={hasScores}
-                        prevSuite={prevSuite}
-                      />
-                    ))}
+                    evaluationWithoutLayoutShift.evals.map((_eval, index) => {
+                      const cacheHitCount =
+                        serverState.cacheHitsByEval[_eval.id] ?? 0;
+                      return (
+                        <EvalTableRow
+                          key={JSON.stringify(_eval.input)}
+                          eval={_eval}
+                          evalIndex={index}
+                          name={name}
+                          timestamp={timestamp}
+                          showExpectedColumn={showExpectedColumn}
+                          isRunningEval={isRunningEval}
+                          hasScores={hasScores}
+                          prevSuite={prevSuite}
+                          cacheHitCount={cacheHitCount}
+                        />
+                      );
+                    })}
               </TableBody>
             </Table>
           </>
diff --git a/apps/evalite-ui/app/sdk.ts b/apps/evalite-ui/app/sdk.ts
@@ -1,7 +1,6 @@
 import { notFound } from "@tanstack/react-router";
 import type { Evalite } from "evalite/types";
-
-const BASE_URL = window.location.origin;
+import { BASE_URL } from "./constants";
 
 declare global {
   interface Window {
diff --git a/packages/evalite/src/evalite.ts b/packages/evalite/src/evalite.ts
@@ -208,6 +208,7 @@ function registerEvalite<TInput, TOutput, TExpected>(
               output: datasetResult.error,
               scores: [],
               traces: [],
+              cacheHits: [],
               renderedColumns: [],
             },
           })
@@ -362,6 +363,7 @@ function registerEvalite<TInput, TOutput, TExpected>(
                 output: serializableOutput,
                 scores,
                 traces: tracesWithFiles,
+                cacheHits,
                 status: "success",
                 renderedColumns,
                 variantName: vitestOpts.variantName,
@@ -397,6 +399,7 @@ function registerEvalite<TInput, TOutput, TExpected>(
                 output: serializedError,
                 scores: [],
                 traces: await handleFilesInTraces(rootDir, traces),
+                cacheHits,
                 status: "fail",
                 renderedColumns: [],
                 variantName: vitestOpts.variantName,
diff --git a/packages/evalite/src/export-static.ts b/packages/evalite/src/export-static.ts
@@ -232,6 +232,7 @@ export const exportStaticUI = async (
   // Generate server-state.json
   const serverState: Evalite.ServerState = {
     type: "idle",
+    cacheHitsByEval: {},
   };
   await fs.writeFile(
     path.join(dataDir, "server-state.json"),
diff --git a/packages/evalite/src/reporter.ts b/packages/evalite/src/reporter.ts
@@ -368,6 +368,7 @@ export default class EvaliteReporter implements Reporter {
           output: null,
           scores: [],
           traces: [],
+          cacheHits: [],
           status: "fail",
           renderedColumns: [],
           variantName: data.initialEval.variantName,
diff --git a/packages/evalite/src/reporter/EvaliteRunner.ts b/packages/evalite/src/reporter/EvaliteRunner.ts
@@ -10,7 +10,7 @@ export interface EvaliteRunnerOptions {
 
 export class EvaliteRunner {
   private opts: EvaliteRunnerOptions;
-  private state: Evalite.ServerState = { type: "idle" };
+  private state: Evalite.ServerState = { type: "idle", cacheHitsByEval: {} };
   private didLastRunFailThreshold: "yes" | "no" | "unknown" = "unknown";
   private collectedResults: Map<string, Evalite.Eval> = new Map();
   private eventQueue: Promise<void> = Promise.resolve();
@@ -98,7 +98,10 @@ export class EvaliteRunner {
       case "running":
         switch (event.type) {
           case "RUN_ENDED":
-            this.updateState({ type: "idle" });
+            this.updateState({
+              type: "idle",
+              cacheHitsByEval: this.state.cacheHitsByEval,
+            });
             break;
           case "EVAL_STARTED":
             {
@@ -247,6 +250,14 @@ export class EvaliteRunner {
                 });
               }
 
+              // Count cache hits for this eval
+              const cacheHitCount = event.eval.cacheHits.filter(
+                (hit) => hit.hit
+              ).length;
+              if (cacheHitCount > 0) {
+                this.state.cacheHitsByEval[evalId] = cacheHitCount;
+              }
+
               const allEvals = await this.opts.storage.evals.getMany({
                 suiteIds: [suite.id],
               });
@@ -295,6 +306,7 @@ export class EvaliteRunner {
           case "RUN_BEGUN":
             // Clear collected results for new run
             this.collectedResults.clear();
+            this.state.cacheHitsByEval = {};
 
             this.updateState({
               filepaths: event.filepaths,
@@ -303,6 +315,7 @@ export class EvaliteRunner {
               runId: undefined, // Run is created lazily
               suiteNamesRunning: [],
               evalIdsRunning: [],
+              cacheHitsByEval: this.state.cacheHitsByEval,
             });
             break;
         }
diff --git a/packages/evalite/src/server.ts b/packages/evalite/src/server.ts
@@ -11,15 +11,18 @@ export type Server = ReturnType<typeof createServer>;
 
 const THROTTLE_TIME = 100;
 
+const INITIAL_STATE: Evalite.IdleServerState = {
+  type: "idle",
+  cacheHitsByEval: {},
+};
+
 export const handleWebsockets = (server: fastify.FastifyInstance) => {
   const websocketListeners = new Map<
     string,
     (event: Evalite.ServerState) => void
   >();
 
-  let currentState: Evalite.ServerState = {
-    type: "idle",
-  };
+  let currentState: Evalite.ServerState = INITIAL_STATE;
 
   let timeout: NodeJS.Timeout | undefined;
 
diff --git a/packages/evalite/src/types.ts b/packages/evalite/src/types.ts
@@ -165,20 +165,24 @@ export declare namespace Evalite {
 
   export type RunType = "full" | "partial";
 
-  export type RunningServerState = {
+  export interface SharedServerState {
+    cacheHitsByEval: Record<number, number>;
+  }
+
+  export interface RunningServerState extends SharedServerState {
     type: "running";
     runType: RunType;
     filepaths: string[];
     runId: number | bigint | undefined;
     suiteNamesRunning: string[];
     evalIdsRunning: (number | bigint)[];
-  };
+  }
 
-  export type ServerState =
-    | RunningServerState
-    | {
-        type: "idle";
-      };
+  export interface IdleServerState extends SharedServerState {
+    type: "idle";
+  }
+
+  export type ServerState = RunningServerState | IdleServerState;
 
   export type MaybePromise<T> = T | Promise<T>;
 
@@ -220,6 +224,7 @@ export declare namespace Evalite {
     scores: Score[];
     duration: number;
     traces: Trace[];
+    cacheHits: Array<{ keyHash: string; hit: boolean; savedDuration: number }>;
     renderedColumns: RenderedColumn[];
   }
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+export const BASE_URL = import.meta.env.DEV`
	`2`	`+ ? "http://localhost:3006"`
	`3`	`+ : window.location.origin;`