diff --git a/ui/src/pages/admin/evaluation/dataset-card.tsx b/ui/src/pages/admin/evaluation/dataset-card.tsx new file mode 100644 index 000000000..852a6d551 --- /dev/null +++ b/ui/src/pages/admin/evaluation/dataset-card.tsx @@ -0,0 +1,222 @@ +import { useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { Play, Trash2, Upload } from "lucide-react"; +import { + createEvalDataset, + deleteEvalDataset, + listEvalDatasets, + startEvalRun, +} from "@/lib/api/evaluation"; +import { ConfirmDialog } from "@/components/shared/confirm-dialog"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; + +const CSV_HINT = "question,expected_answer,expected_file_ids"; + +export function DatasetCard({ runActive }: { runActive: boolean }) { + const queryClient = useQueryClient(); + const [uploadOpen, setUploadOpen] = useState(false); + + const { data, isLoading } = useQuery({ queryKey: ["eval-datasets"], queryFn: listEvalDatasets }); + + const startMut = useMutation({ + mutationFn: (datasetId: string) => startEvalRun(datasetId), + onSuccess: () => { + toast.success("Evaluation run queued"); + queryClient.invalidateQueries({ queryKey: ["eval-runs"] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const deleteMut = useMutation({ + mutationFn: (id: string) => deleteEvalDataset(id), + onSuccess: () => { + toast.success("Dataset deleted"); + queryClient.invalidateQueries({ queryKey: ["eval-datasets"] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const datasets = data ?? []; + + return ( + + +
+ Datasets + + A corpus to index plus a CSV test set ({CSV_HINT}). + +
+ + + + {isLoading ? ( + + ) : datasets.length === 0 ? ( +

+ No datasets yet. Upload a corpus and a test set to run an evaluation. +

+ ) : ( + + + + Name + Files + Questions + + + + + {datasets.map((dataset) => ( + + {dataset.name} + {dataset.corpus_file_count} + {dataset.testset_row_count} + + + deleteMut.mutate(dataset.id)} + > + + + + + ))} + +
+ )} + + + + + ); +} + +function UploadDialog({ + open, + onOpenChange, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const queryClient = useQueryClient(); + const [name, setName] = useState(""); + const [testset, setTestset] = useState(null); + const [corpus, setCorpus] = useState([]); + + const reset = () => { + setName(""); + setTestset(null); + setCorpus([]); + }; + + const createMut = useMutation({ + mutationFn: () => createEvalDataset(name, testset as File, corpus), + onSuccess: (dataset) => { + toast.success(`Dataset "${dataset.name}" created (${dataset.testset_row_count} questions)`); + queryClient.invalidateQueries({ queryKey: ["eval-datasets"] }); + reset(); + onOpenChange(false); + }, + // The API validates the CSV row by row; surface its message verbatim. + onError: (e) => toast.error((e as Error).message), + }); + + const canSubmit = name.trim() !== "" && testset !== null && corpus.length > 0; + + // Every dismissal path — backdrop, Escape, Cancel — clears the form, so a + // reopened dialog never silently resubmits the previous selection. + const close = () => { + reset(); + onOpenChange(false); + }; + + return ( + (next ? onOpenChange(true) : close())}> + + + New evaluation dataset + + The corpus is re-indexed on every run, which is what the indexing-speed numbers measure. + + + +
+
+ + setName(e.target.value)} + placeholder="Support docs — July" + /> +
+
+ + setTestset(e.target.files?.[0] ?? null)} + /> +

+ Header: {CSV_HINT}. expected_file_ids is optional and + semicolon-separated; rows without it are excluded from hit rate, MRR and recall. +

+
+
+ + setCorpus(Array.from(e.target.files ?? []))} + /> + {corpus.length > 0 && ( +

{corpus.length} file(s) selected

+ )} +
+
+ + + + + + + + ); +} diff --git a/ui/src/pages/admin/evaluation/run-detail.tsx b/ui/src/pages/admin/evaluation/run-detail.tsx new file mode 100644 index 000000000..7e9f7c968 --- /dev/null +++ b/ui/src/pages/admin/evaluation/run-detail.tsx @@ -0,0 +1,240 @@ +import { useQuery } from "@tanstack/react-query"; +import { EVAL_POLL_MS, getEvalRun, isActiveStatus, type EvalRun } from "@/lib/api/evaluation"; +import { StatusBadge } from "@/components/shared/status-badge"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; + +function percent(value: number | null | undefined): string { + return value === null || value === undefined ? "—" : `${(value * 100).toFixed(1)}%`; +} + +function score(value: number | null | undefined): string { + return value === null || value === undefined ? "—" : value.toFixed(3); +} + +function Stat({ label, value, hint }: { label: string; value: string; hint?: string }) { + return ( +
+
{label}
+
{value}
+ {hint &&

{hint}

} +
+ ); +} + +export function RunDetail({ runId }: { runId: string }) { + const { data: run, isLoading } = useQuery({ + queryKey: ["eval-run", runId], + queryFn: () => getEvalRun(runId), + // Poll only while the run can still change. + refetchInterval: (query) => { + const status = query.state.data?.status; + return status && isActiveStatus(status) ? EVAL_POLL_MS : false; + }, + }); + + if (isLoading) return ; + if (!run) return null; + + return ( +
+ + {run.error && ( + + {run.error} + + )} + + + +
+ ); +} + +function RunHeader({ run }: { run: EvalRun }) { + const elapsed = + run.started_at && run.finished_at + ? `${Math.round( + (new Date(run.finished_at).getTime() - new Date(run.started_at).getTime()) / 1000, + )}s` + : "—"; + + return ( + + +
+ {run.id.slice(0, 12)} + + Started {run.started_at ? new Date(run.started_at).toLocaleString() : "—"} · took {elapsed} + +
+ + + + ); +} + +function IndexingPanel({ run }: { run: EvalRun }) { + const metrics = run.indexing; + return ( + + + Indexing speed + End-to-end ingestion of the corpus into a throwaway partition. + + + {!metrics ? ( +

Not measured yet.

+ ) : ( + <> +
+ + + + + + 0 ? `${metrics.files_failed} failed` : undefined} + /> + +
+ {Object.keys(metrics.by_extension).length > 1 && ( +
+ {Object.entries(metrics.by_extension).map(([extension, bucket]) => ( + + {extension} · {bucket.files} file(s) ·{" "} + {bucket.mean_seconds}s avg + + ))} +
+ )} + + )} + + + ); +} + +function QualityPanel({ run }: { run: EvalRun }) { + const retrieval = run.retrieval; + const answer = run.answer; + + return ( +
+ + + Retrieval quality + + {retrieval + ? `${retrieval.scored_cases} question(s) scored${ + retrieval.skipped_cases > 0 + ? `, ${retrieval.skipped_cases} skipped (no expected_file_ids)` + : "" + }` + : "Not measured yet."} + + + + {retrieval && ( +
+ + + + +
+ )} + + + + + + Answer quality + + {answer ? `${answer.scored_cases} answer(s) graded by the LLM` : "Not measured yet."} + + + + {answer && ( +
+ + + +
+ )} + + +
+ ); +} + +function CasesTable({ run }: { run: EvalRun }) { + if (run.cases.length === 0) return null; + + return ( + + + Questions + + +
+ + + + Question + Hit + RR + Answer + Grader + + + + {run.cases.map((testCase, index) => ( + + +

+ {testCase.query} +

+ {testCase.retrieved_file_ids.length > 0 && ( +

+ {testCase.retrieved_file_ids.join(", ")} +

+ )} + + + {testCase.hit === null ? ( + — + ) : ( + + )} + + + {testCase.reciprocal_rank === null ? "—" : testCase.reciprocal_rank.toFixed(2)} + + + {testCase.answer_passed === null ? ( + — + ) : ( + + )} + + +

+ {testCase.grader_reason ?? "—"} +

+ + + ))} + +
+
+ + + ); +}