feat(evaluation) 14/15: Evaluation tab on the admin System page

ui/src/pages/admin/evaluation/index.test.tsx

@@ -0,0 +1,212 @@
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { render, screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  cancelEvalRun,
+  getEvalRun,
+  listEvalDatasets,
+  listEvalRuns,
+  startEvalRun,
+  type EvalRun,
+  type EvalRunSummary,
+} from "@/lib/api/evaluation";
+import { EvaluationTab } from "./index";
+
+vi.mock("sonner", () => ({
+  toast: { success: vi.fn(), error: vi.fn() },
+}));
+
+vi.mock("@/lib/api/evaluation", async () => {
+  const actual = await vi.importActual<typeof import("@/lib/api/evaluation")>("@/lib/api/evaluation");
+  return {
+    ...actual,
+    listEvalDatasets: vi.fn(),
+    listEvalRuns: vi.fn(),
+    getEvalRun: vi.fn(),
+    startEvalRun: vi.fn(),
+    cancelEvalRun: vi.fn(),
+    deleteEvalDataset: vi.fn(),
+    createEvalDataset: vi.fn(),
+  };
+});
+
+const listDatasetsMock = vi.mocked(listEvalDatasets);
+const listRunsMock = vi.mocked(listEvalRuns);
+const getRunMock = vi.mocked(getEvalRun);
+const startRunMock = vi.mocked(startEvalRun);
+const cancelRunMock = vi.mocked(cancelEvalRun);
+
+const DATASET = {
+  id: "ds1",
+  name: "Support docs",
+  corpus_file_count: 3,
+  testset_row_count: 12,
+  created_at: null,
+  created_by: 1,
+};
+
+const COMPLETED_RUN: EvalRunSummary = {
+  id: "run-completed",
+  dataset_id: "ds1",
+  status: "COMPLETED",
+  started_at: "2026-07-27T10:00:00Z",
+  finished_at: "2026-07-27T10:05:00Z",
+  hit_rate: 0.75,
+  mrr: 0.5,
+  answer_pass_rate: 1,
+  files_per_minute: 12.5,
+  error: null,
+};
+
+const RUN_DETAIL: EvalRun = {
+  id: "run-completed",
+  dataset_id: "ds1",
+  status: "COMPLETED",
+  started_at: "2026-07-27T10:00:00Z",
+  finished_at: "2026-07-27T10:05:00Z",
+  indexing: {
+    files_total: 3,
+    files_failed: 0,
+    bytes_total: 3 * 1024 * 1024,
+    wall_seconds: 14.4,
+    files_per_minute: 12.5,
+    megabytes_per_second: 0.21,
+    p50_seconds: 4.5,
+    p95_seconds: 6.1,
+    by_extension: {},
+    samples: [],
+  },
+  retrieval: {
+    scored_cases: 8,
+    skipped_cases: 4,
+    hit_rate: 0.75,
+    mrr: 0.5,
+    recall: 0.6,
+    context_relevance: 0.82,
+  },
+  answer: { scored_cases: 12, pass_rate: 1, factuality: 0.9, rubric_score: 0.85 },
+  cases: [
+    {
+      query: "What is the refund window?",
+      retrieved_file_ids: ["policy.pdf"],
+      expected_file_ids: ["policy.pdf"],
+      hit: true,
+      reciprocal_rank: 1,
+      answer: "30 days",
+      answer_passed: true,
+      grader_reason: "Matches the reference answer",
+    },
+  ],
+  error: null,
+  created_by: 1,
+};
+
+function renderTab() {
+  const client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+  return render(
+    <QueryClientProvider client={client}>
+      <EvaluationTab />
+    </QueryClientProvider>,
+  );
+}
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  listDatasetsMock.mockResolvedValue([DATASET]);
+  listRunsMock.mockResolvedValue([COMPLETED_RUN]);
+  getRunMock.mockResolvedValue(RUN_DETAIL);
+});
+
+describe("EvaluationTab", () => {
+  it("lists datasets with their corpus and question counts", async () => {
+    renderTab();
+    expect(await screen.findByText("Support docs")).toBeTruthy();
+    expect(screen.getByText("3")).toBeTruthy();
+    expect(screen.getByText("12")).toBeTruthy();
+  });
+
+  it("starts a run for the chosen dataset", async () => {
+    startRunMock.mockResolvedValue(RUN_DETAIL);
+    renderTab();
+
+    await userEvent.click(await screen.findByRole("button", { name: /^run$/i }));
+
+    await waitFor(() => expect(startRunMock).toHaveBeenCalledWith("ds1"));
+  });
+
+  it("disables starting a run while one is in flight", async () => {
+    listRunsMock.mockResolvedValue([{ ...COMPLETED_RUN, id: "run-active", status: "INDEXING" }]);
+    getRunMock.mockResolvedValue({ ...RUN_DETAIL, id: "run-active", status: "INDEXING" });
+    renderTab();
+
+    await waitFor(() =>
+      expect(screen.getByRole("button", { name: /^run$/i }).hasAttribute("disabled")).toBe(true),
+    );
+  });
+
+  it("offers cancel only while a run is active", async () => {
+    renderTab();
+    await screen.findByText("Support docs");
+    expect(screen.queryByRole("button", { name: /cancel run/i })).toBeNull();
+
+    listRunsMock.mockResolvedValue([{ ...COMPLETED_RUN, id: "run-active", status: "EVALUATING" }]);
+    getRunMock.mockResolvedValue({ ...RUN_DETAIL, id: "run-active", status: "EVALUATING" });
+    cancelRunMock.mockResolvedValue({ ...RUN_DETAIL, status: "CANCELLED" });
+
+    const { unmount } = renderTab();
+    const cancelButton = await screen.findAllByRole("button", { name: /cancel run/i });
+    await userEvent.click(cancelButton[0]);
+    await waitFor(() => expect(cancelRunMock).toHaveBeenCalledWith("run-active"));
+    unmount();
+  });
+
+  it("shows the three metric families for the selected run", async () => {
+    renderTab();
+
+    expect(await screen.findByText("Indexing speed")).toBeTruthy();
+    expect(screen.getByText("Retrieval quality")).toBeTruthy();
+    expect(screen.getByText("Answer quality")).toBeTruthy();
+    // hit_rate 0.75 rendered as a percentage in the detail panel
+    expect(screen.getByText("75.0%")).toBeTruthy();
+    // Throughput appears twice: once in the run row, once as a detail stat.
+    expect(screen.getAllByText("12.5").length).toBeGreaterThan(0);
+  });
+
+  it("reports how many questions were skipped for lacking ground-truth sources", async () => {
+    renderTab();
+    expect(
+      await screen.findByText(/8 question\(s\) scored, 4 skipped \(no expected_file_ids\)/),
+    ).toBeTruthy();
+  });
+
+  it("renders the per-question table with the grader's reasoning", async () => {
+    renderTab();
+    expect(await screen.findByText("What is the refund window?")).toBeTruthy();
+    expect(screen.getByText("Matches the reference answer")).toBeTruthy();
+  });
+
+  it("surfaces a failed run's error message", async () => {
+    listRunsMock.mockResolvedValue([
+      { ...COMPLETED_RUN, id: "run-failed", status: "FAILED", error: "promptfoo timed out." },
+    ]);
+    getRunMock.mockResolvedValue({
+      ...RUN_DETAIL,
+      id: "run-failed",
+      status: "FAILED",
+      error: "promptfoo timed out.",
+    });
+    renderTab();
+
+    expect(await screen.findByText("promptfoo timed out.")).toBeTruthy();
+  });
+
+  it("tells the admin what to do when there are no datasets yet", async () => {
+    listDatasetsMock.mockResolvedValue([]);
+    listRunsMock.mockResolvedValue([]);
+    renderTab();
+
+    expect(await screen.findByText(/No datasets yet/)).toBeTruthy();
+    expect(screen.getByText("No runs yet.")).toBeTruthy();
+  });
+});

ui/src/pages/admin/evaluation/index.tsx

@@ -0,0 +1,133 @@
+import { useState } from "react";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { toast } from "sonner";
+import { Ban } from "lucide-react";
+import { EVAL_POLL_MS, cancelEvalRun, isActiveStatus, listEvalRuns } from "@/lib/api/evaluation";
+import { StatusBadge } from "@/components/shared/status-badge";
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
+import { Skeleton } from "@/components/ui/skeleton";
+import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
+import { DatasetCard } from "./dataset-card";
+import { RunDetail } from "./run-detail";
+
+function percent(value: number | null): string {
+  return value === null ? "—" : `${(value * 100).toFixed(0)}%`;
+}
+
+/**
+ * Evaluation tab: upload a dataset, run it, read the numbers.
+ *
+ * Runs are serialised server-side (one at a time), so the run list drives
+ * both the polling cadence and whether a new run can be started.
+ */
+export function EvaluationTab() {
+  const queryClient = useQueryClient();
+  const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
+
+  const { data, isLoading } = useQuery({
+    queryKey: ["eval-runs"],
+    queryFn: () => listEvalRuns(),
+    refetchInterval: (query) =>
+      (query.state.data ?? []).some((run) => isActiveStatus(run.status)) ? EVAL_POLL_MS : false,
+  });
+
+  const cancelMut = useMutation({
+    mutationFn: (id: string) => cancelEvalRun(id),
+    onSuccess: () => {
+      toast.success("Cancellation requested");
+      queryClient.invalidateQueries({ queryKey: ["eval-runs"] });
+    },
+    onError: (e) => toast.error((e as Error).message),
+  });
+
+  const runs = data ?? [];
+  const activeRun = runs.find((run) => isActiveStatus(run.status)) ?? null;
+  // Default to the newest run so the tab is not empty after a run finishes.
+  const shownRunId = selectedRunId ?? activeRun?.id ?? runs[0]?.id ?? null;
+
+  return (
+    <div className="space-y-4">
+      <DatasetCard runActive={activeRun !== null} />
+
+      <Card>
+        <CardHeader className="flex flex-row items-start justify-between">
+          <div>
+            <CardTitle>Runs</CardTitle>
+            <CardDescription>
+              One run at a time, so indexing timings stay comparable between them.
+            </CardDescription>
+          </div>
+          {activeRun && (
+            <Button
+              size="sm"
+              variant="outline"
+              disabled={cancelMut.isPending}
+              onClick={() => cancelMut.mutate(activeRun.id)}
+            >
+              <Ban className="mr-2 h-4 w-4" />
+              Cancel run
+            </Button>
+          )}
+        </CardHeader>
+        <CardContent>
+          {isLoading ? (
+            <Skeleton className="h-24" />
+          ) : runs.length === 0 ? (
+            <p className="py-6 text-center text-sm text-muted-foreground">No runs yet.</p>
+          ) : (
+            <Table>
+              <TableHeader>
+                <TableRow>
+                  <TableHead>Started</TableHead>
+                  <TableHead>Status</TableHead>
+                  <TableHead className="text-right">Files/min</TableHead>
+                  <TableHead className="text-right">Hit rate</TableHead>
+                  <TableHead className="text-right">MRR</TableHead>
+                  <TableHead className="text-right">Answers</TableHead>
+                </TableRow>
+              </TableHeader>
+              <TableBody>
+                {runs.map((run) => (
+                  <TableRow
+                    key={run.id}
+                    className={`cursor-pointer ${run.id === shownRunId ? "bg-muted/50" : ""}`}
+                    role="button"
+                    tabIndex={0}
+                    aria-current={run.id === shownRunId}
+                    onClick={() => setSelectedRunId(run.id)}
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter" || e.key === " ") {
+                        e.preventDefault();
+                        setSelectedRunId(run.id);
+                      }
+                    }}
+                  >
+                    <TableCell>
+                      {run.started_at ? new Date(run.started_at).toLocaleString() : "—"}
+                    </TableCell>
+                    <TableCell>
+                      <StatusBadge status={run.status} />
+                    </TableCell>
+                    <TableCell className="text-right tabular-nums">
+                      {run.files_per_minute?.toFixed(1) ?? "—"}
+                    </TableCell>
+                    <TableCell className="text-right tabular-nums">{percent(run.hit_rate)}</TableCell>
+                    <TableCell className="text-right tabular-nums">
+                      {run.mrr?.toFixed(2) ?? "—"}
+                    </TableCell>
+                    <TableCell className="text-right tabular-nums">
+                      {percent(run.answer_pass_rate)}
+                    </TableCell>
+                  </TableRow>
+                ))}
+              </TableBody>
+            </Table>
+          )}
+        </CardContent>
+      </Card>
+
+      {shownRunId && <RunDetail runId={shownRunId} />}
+    </div>
+  );
+}

ui/src/pages/admin/system.tsx

@@ -15,6 +15,7 @@ import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import { Badge } from "@/components/ui/badge";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import { EvaluationTab } from "./evaluation";
 import { Skeleton } from "@/components/ui/skeleton";
 
 const GRAFANA_URL = import.meta.env.VITE_GRAFANA_URL || "";
@@ -31,7 +32,7 @@ export default function SystemPage() {
     <div>
       <PageHeader
         title="System"
-        description="Status, Ray actors, metrics and configuration"
+        description="Status, Ray actors, metrics, configuration and evaluation"
         actions={
           GRAFANA_URL ? (
             <Button variant="outline" asChild>
@@ -50,6 +51,7 @@ export default function SystemPage() {
           <TabsTrigger value="actors">Actors</TabsTrigger>
           <TabsTrigger value="metrics">Metrics</TabsTrigger>
           <TabsTrigger value="config">Config</TabsTrigger>
+          <TabsTrigger value="evaluation">Evaluation</TabsTrigger>
         </TabsList>
 
         <TabsContent value="status">
@@ -64,6 +66,9 @@ export default function SystemPage() {
         <TabsContent value="config">
           <ConfigTab />
         </TabsContent>
+        <TabsContent value="evaluation">
+          <EvaluationTab />
+        </TabsContent>
       </Tabs>
     </div>
   );