From 0ef8bb34f09f0f34c56f48c00a9d9f3900a46e94 Mon Sep 17 00:00:00 2001 From: Sean Teramae Date: Wed, 19 Aug 2026 09:58:47 -0700 Subject: [PATCH] feat(studio): Experiments Panel for Agent Details Signed-off-by: Sean Teramae --- web/packages/studio/src/constants/links.ts | 2 +- web/packages/studio/src/mocks/handlers.ts | 101 ++++++------ .../agents/AgentDetailRoute/OverviewTab.tsx | 45 ++++-- .../agents/AgentDetailRoute/index.test.tsx | 4 +- .../routes/agents/AgentDetailRoute/index.tsx | 2 + .../RecentExperimentsPanel.stories.tsx | 135 ++++++++++++++++ .../overview/RecentExperimentsPanel.test.tsx | 101 ++++++++++++ .../overview/RecentExperimentsPanel.tsx | 95 ++++++++++++ .../overview/recentExperiments.test.ts | 136 ++++++++++++++++ .../overview/recentExperiments.ts | 146 ++++++++++++++++++ .../AgentDetailRoute/useAgentDetails.ts | 33 ++-- 11 files changed, 723 insertions(+), 77 deletions(-) create mode 100644 web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.stories.tsx create mode 100644 web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.test.tsx create mode 100644 web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.tsx create mode 100644 web/packages/studio/src/routes/agents/AgentDetailRoute/overview/recentExperiments.test.ts create mode 100644 web/packages/studio/src/routes/agents/AgentDetailRoute/overview/recentExperiments.ts diff --git a/web/packages/studio/src/constants/links.ts b/web/packages/studio/src/constants/links.ts index 3f73ff0232..e235116706 100644 --- a/web/packages/studio/src/constants/links.ts +++ b/web/packages/studio/src/constants/links.ts @@ -54,4 +54,4 @@ export const LINK_DOCS_JOBS = `${DOCS_BASE_URL}studio#jobs`; export const LINK_DOCS_SECRETS = `${DOCS_BASE_URL}get-started/core-concepts/manage-secrets`; // Evaluations -export const LINK_DOCS_EXPERIMENTS_CLI = `${DOCS_BASE_URL}reference/cli-reference`; +export const LINK_DOCS_EXPERIMENTS_CLI = `${DOCS_BASE_URL}evaluate-models/experiments`; diff --git a/web/packages/studio/src/mocks/handlers.ts b/web/packages/studio/src/mocks/handlers.ts index 54eb42068c..0f57996a5e 100644 --- a/web/packages/studio/src/mocks/handlers.ts +++ b/web/packages/studio/src/mocks/handlers.ts @@ -68,6 +68,47 @@ export interface HypermodelParams { hypermodelId: string; } +/** Shared by the agents list and get-by-name handlers so both agree on the same fixtures. */ +const mockAgents = (workspace: unknown) => { + const config = { + functions: { wiki: { _type: 'wiki_search' }, clock: { _type: 'current_datetime' } }, + llms: { + llm: { + _type: 'openai', + api_key: 'not-used', + model_name: 'meta-llama-3-1-70b-instruct', + temperature: 0, + }, + }, + workflow: { + _type: 'react_agent', + tool_names: ['wiki', 'clock'], + llm_name: 'llm', + verbose: false, + parse_agent_response_max_retries: 3, + }, + }; + + return [ + { + name: 'react-agent', + workspace, + description: '', + created_at: '2026-04-20T10:00:00Z', + config, + config_format: 'nat-workflow-v1', + }, + { + name: 'react-agent2', + workspace, + description: 'Second react agent', + created_at: '2026-04-22T10:00:00Z', + config, + config_format: 'nat-workflow-v1', + }, + ]; +}; + /** * Happy path handlers for all UI tests. They usually return mock fixtures, like example Hypermodel response objects. * Having a single source of happy path MSW handlers is listed in the [MSW docs as a best practice](https://mswjs.io/docs/best-practices/structuring-handlers#handlers-structure), @@ -593,58 +634,7 @@ export const handlers = [ http.get( `${PLATFORM_BASE_URL}/apis/agents/v2/workspaces/:workspace/agents`, ({ params, request }) => { - const data = [ - { - name: 'react-agent', - workspace: params['workspace'], - description: '', - created_at: '2026-04-20T10:00:00Z', - config: { - functions: { wiki: { _type: 'wiki_search' }, clock: { _type: 'current_datetime' } }, - llms: { - llm: { - _type: 'openai', - api_key: 'not-used', - model_name: 'meta-llama-3-1-70b-instruct', - temperature: 0, - }, - }, - workflow: { - _type: 'react_agent', - tool_names: ['wiki', 'clock'], - llm_name: 'llm', - verbose: false, - parse_agent_response_max_retries: 3, - }, - }, - config_format: 'nat-workflow-v1', - }, - { - name: 'react-agent2', - workspace: params['workspace'], - description: 'Second react agent', - created_at: '2026-04-22T10:00:00Z', - config: { - functions: { wiki: { _type: 'wiki_search' }, clock: { _type: 'current_datetime' } }, - llms: { - llm: { - _type: 'openai', - api_key: 'not-used', - model_name: 'meta-llama-3-1-70b-instruct', - temperature: 0, - }, - }, - workflow: { - _type: 'react_agent', - tool_names: ['wiki', 'clock'], - llm_name: 'llm', - verbose: false, - parse_agent_response_max_retries: 3, - }, - }, - config_format: 'nat-workflow-v1', - }, - ]; + const data = mockAgents(params['workspace']); const url = new URL(request.url); const sort = url.searchParams.get('sort') ?? '-created_at'; @@ -677,6 +667,13 @@ export const handlers = [ }); } ), + http.get( + `${PLATFORM_BASE_URL}/apis/agents/v2/workspaces/:workspace/agents/:name`, + ({ params }) => { + const agent = mockAgents(params['workspace']).find((a) => a.name === params['name']); + return agent ? HttpResponse.json(agent) : new HttpResponse(null, { status: 404 }); + } + ), http.delete( `${PLATFORM_BASE_URL}/apis/agents/v2/workspaces/:workspace/agents/:name`, () => new HttpResponse(null, { status: 204 }) diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/OverviewTab.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/OverviewTab.tsx index ef8f663e56..999422c33d 100644 --- a/web/packages/studio/src/routes/agents/AgentDetailRoute/OverviewTab.tsx +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/OverviewTab.tsx @@ -9,28 +9,44 @@ import { bucketAdverbForRange } from '@studio/components/AgentTraceStatistics/ut import { INTAKE_ENABLED, OPTIMIZER_ENABLED } from '@studio/constants/environment'; import { AgentSummaryPanel } from '@studio/routes/agents/AgentDetailRoute/overview/AgentSummaryPanel'; import { OpenInsightsPanel } from '@studio/routes/agents/AgentDetailRoute/overview/OpenInsightsPanel'; +import { toRecentExperiments } from '@studio/routes/agents/AgentDetailRoute/overview/recentExperiments'; +import { RecentExperimentsPanel } from '@studio/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel'; import { useOpenInsights } from '@studio/routes/agents/AgentDetailRoute/overview/useOpenInsights'; import { useOverviewTraces } from '@studio/routes/agents/AgentDetailRoute/overview/useOverviewTraces'; +import type { AgentEvaluationRow } from '@studio/routes/agents/AgentDetailRoute/useAgentDetails'; import { + getExperimentDetailRoute, getIntakeTracesRoute, getOptimizerInsightRoute, getOptimizerRoute, } from '@studio/routes/utils'; -import { type FC, useState } from 'react'; +import { type FC, useMemo, useState } from 'react'; import { useNavigate } from 'react-router'; interface OverviewTabProps { workspace: string; agent?: Agent; modelNames: string[]; + /** The agent's published evaluations, which the experiment cards are rolled up from. */ + evals: AgentEvaluationRow[]; /** Jump to the chat tab so the agent emits its first traces. */ onRunAgent: () => void; + /** Open the submit-evaluation modal from the experiments empty state. */ + onRunEvaluation?: () => void; } /** Landing view for an agent: how it has been running, next to what it is. */ -export const OverviewTab: FC = ({ workspace, agent, modelNames, onRunAgent }) => { +export const OverviewTab: FC = ({ + workspace, + agent, + modelNames, + evals, + onRunAgent, + onRunEvaluation, +}) => { const navigate = useNavigate(); const [range, setRange] = useState('week'); + const experiments = useMemo(() => toRecentExperiments(evals), [evals]); const { traces, isPending } = useOverviewTraces({ workspace, range, enabled: INTAKE_ENABLED }); const { insights, @@ -50,14 +66,23 @@ export const OverviewTab: FC = ({ workspace, agent, modelNames return ( - navigate(getIntakeTracesRoute(workspace))} - onRunAgent={onRunAgent} - isPending={isPending} - caption={`${bucketAdverbForRange(range)} · Workspace-wide`} + {INTAKE_ENABLED && ( + navigate(getIntakeTracesRoute(workspace))} + onRunAgent={onRunAgent} + isPending={isPending} + caption={`${bucketAdverbForRange(range)}`} + /> + )} + + experiment.name && navigate(getExperimentDetailRoute(workspace, experiment.name)) + } + onRunEvaluation={onRunEvaluation} /> diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/index.test.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/index.test.tsx index 0cb448011f..70319c3ef6 100644 --- a/web/packages/studio/src/routes/agents/AgentDetailRoute/index.test.tsx +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/index.test.tsx @@ -38,7 +38,9 @@ describe('AgentDetailRoute', () => { expect(screen.getByRole('tab', { name: 'Details' })).toBeInTheDocument(); expect(screen.queryByRole('tab', { name: 'Configuration' })).not.toBeInTheDocument(); expect(screen.getByRole('button', { name: 'Open traces' })).toBeInTheDocument(); - expect(screen.getByRole('button', { name: 'Run evaluation' })).toBeInTheDocument(); + // The overview's experiments empty state offers the same action, so this label appears twice: + // once in the header, once in that empty state. + expect(screen.getAllByRole('button', { name: 'Run evaluation' })).toHaveLength(2); expect(screen.getByRole('button', { name: 'Deploy' })).toBeInTheDocument(); expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); }); diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/index.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/index.tsx index c64bdabb0f..12e6af7273 100644 --- a/web/packages/studio/src/routes/agents/AgentDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/index.tsx @@ -218,7 +218,9 @@ export const AgentDetailRoute: FC = () => { workspace={workspace} agent={agent} modelNames={modelNames} + evals={agentEvals} onRunAgent={() => setSelectedTab('chat')} + onRunEvaluation={agentName ? () => setSubmitEvalOpen(true) : undefined} /> )} diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.stories.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.stories.tsx new file mode 100644 index 0000000000..fe8c79042a --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.stories.tsx @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { Meta, StoryObj } from '@storybook/react'; +import { toRecentExperiments } from '@studio/routes/agents/AgentDetailRoute/overview/recentExperiments'; +import { RecentExperimentsPanel } from '@studio/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel'; +import type { AgentEvaluationRow } from '@studio/routes/agents/AgentDetailRoute/useAgentDetails'; + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** Fixed rather than `Date.now()` so the stories render identically every run. */ +const LATEST = Date.parse('2026-08-18T00:00:00Z'); + +interface RunOptions { + experimentId: string; + experimentName: string; + experimentDescription: string; + /** How long before {@link LATEST} the run published. */ + daysAgo: number; + scores: Record; +} + +/** Fixtures are pushed through the real `toRecentExperiments`, so the stories exercise the actual + * derivation (series ordering, delta window) rather than hand-built props. */ +const run = ({ + experimentId, + experimentName, + experimentDescription, + daysAgo, + scores, +}: RunOptions): AgentEvaluationRow => + ({ + id: `${experimentId}-${daysAgo}`, + name: `${experimentId}-run-${daysAgo}`, + workspace: 'default', + experiment_ids: [experimentId], + dataset_name: 'support-bench-v3', + experimentName, + experimentDescription, + created_at: new Date(LATEST - daysAgo * DAY_MS).toISOString(), + aggregate_scores: Object.fromEntries( + Object.entries(scores).map(([key, mean]) => [key, { mean }]) + ), + }) as AgentEvaluationRow; + +/** A run per week going back, each evaluator drifting by a fixed step so the trend is legible. */ +const weeklyRuns = ( + experiment: Omit, + evaluators: Record, + weeks = 8 +): AgentEvaluationRow[] => + Array.from({ length: weeks }, (_, index) => + run({ + ...experiment, + daysAgo: index * 7, + scores: Object.fromEntries( + Object.entries(evaluators).map(([name, { from, step }]) => [ + name, + Number((from - index * step).toFixed(3)), + ]) + ), + }) + ); + +const evaluations: AgentEvaluationRow[] = [ + ...weeklyRuns( + { + experimentId: 'exp-v2', + experimentName: 'v2 use cases', + experimentDescription: + 'Dataset of early v2 use cases to support feature development ahead of the release.', + }, + { solved: { from: 0.16, step: 0.011 }, helpfulness: { from: 0.84, step: 0.02 } } + ), + ...weeklyRuns( + { + experimentId: 'exp-primary', + experimentName: 'Primary use cases', + experimentDescription: + 'Continuously evaluate every merge to main against the full Support-Bench v3 benchmark.', + }, + { + solved: { from: 0.78, step: 0.02 }, + // Negative step: this one has regressed over the window, so its delta renders red and down. + 'llm-judge.tone': { from: 0.75, step: -0.01 }, + } + ), +]; + +const meta: Meta = { + title: 'Studio/RecentExperimentsPanel', + component: RecentExperimentsPanel, + args: { + experiments: toRecentExperiments(evaluations), + onOpenExperiment: () => {}, + onRunEvaluation: () => {}, + }, + decorators: [ + (Story) => ( +
+ +
+ ), + ], +}; + +export default meta; + +type Story = StoryObj; + +export const Default: Story = {}; + +/** A brand-new experiment: one published run, so there is a score but no trend and no delta. */ +export const SingleRun: Story = { + args: { + experiments: toRecentExperiments([ + run({ + experimentId: 'exp-primary', + experimentName: 'Primary use cases', + experimentDescription: + 'Continuously evaluate every merge to main against the full Support-Bench v3 benchmark.', + daysAgo: 0, + scores: { solved: 0.78, accuracy: 0.91, 'llm-judge.tone': 0.75 }, + }), + ]), + }, +}; + +export const Empty: Story = { + args: { experiments: [] }, +}; + +export const Loading: Story = { + args: { isPending: true }, +}; diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.test.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.test.tsx new file mode 100644 index 0000000000..2db91b540f --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.test.tsx @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RecentExperiment } from '@studio/routes/agents/AgentDetailRoute/overview/recentExperiments'; +import { RecentExperimentsPanel } from '@studio/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel'; +import { render, screen } from '@studio/tests/util/render'; +import userEvent from '@testing-library/user-event'; + +const experiment: RecentExperiment = { + id: 'exp-1', + name: 'v2 use cases', + description: 'Dataset of early v2 use cases.', + latestCreatedAt: '2026-08-10T00:00:00Z', + evaluationCount: 3, + series: [ + { + id: 'solved', + label: 'Solved', + value: 0.16, + // Already a relative change, in percent, as `toRecentExperiments` produces it. + delta: 7.38, + points: [ + { label: 'Aug 1', value: 0.1 }, + { label: 'Aug 10', value: 0.16 }, + ], + }, + { + id: 'tool_use', + label: 'Tool Use', + value: 0.9, + points: [{ label: 'Aug 10', value: 0.9 }], + }, + ], +}; + +describe('RecentExperimentsPanel', () => { + it('renders a trend card per experiment with its latest score and delta', () => { + render(); + + expect(screen.getByText('Recent experiments')).toBeInTheDocument(); + expect(screen.getByText('v2 use cases')).toBeInTheDocument(); + expect(screen.getByText('Dataset of early v2 use cases.')).toBeInTheDocument(); + // The score is a bare float (no scale metadata); the delta is a relative change, so it is a + // percentage regardless of that scale. + expect(screen.getByText('0.16')).toBeInTheDocument(); + expect(screen.getByText('+7.4%')).toBeInTheDocument(); + expect(screen.getByText('vs. 7 days ago')).toBeInTheDocument(); + }); + + it('switches the displayed value when another evaluator is selected', async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText('Tool Use')); + + expect(screen.getByText('0.9')).toBeInTheDocument(); + // The selected evaluator has no week-old baseline, so no delta is claimed. + expect(screen.queryByText('vs. 7 days ago')).not.toBeInTheDocument(); + }); + + it('opens the experiment from its View action', async () => { + const user = userEvent.setup(); + const onOpenExperiment = vi.fn(); + render( + + ); + + await user.click(screen.getByRole('button', { name: 'View' })); + + expect(onOpenExperiment).toHaveBeenCalledWith(experiment); + }); + + it('offers no View action for an experiment whose name never resolved', () => { + render( + + ); + + expect(screen.getByText('Unnamed experiment')).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: 'View' })).not.toBeInTheDocument(); + }); + + it('prompts for a first evaluation when the agent has no experiments', async () => { + const user = userEvent.setup(); + const onRunEvaluation = vi.fn(); + render( + + ); + + expect(screen.getByText('No experiments')).toBeInTheDocument(); + await user.click(screen.getByRole('button', { name: 'Run evaluation' })); + + expect(onRunEvaluation).toHaveBeenCalled(); + }); +}); diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.tsx new file mode 100644 index 0000000000..76e2ec1c17 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/RecentExperimentsPanel.tsx @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { formatEvaluatorScore } from '@nemo/common/src/utils/formatters'; +import { Button, Card, Flex, Stack, StatusMessage, Text } from '@nvidia/foundations-react-core'; +import { MetricTrendPanel } from '@studio/components/charts/MetricTrendPanel'; +import { StackedSkeleton } from '@studio/components/StackedSkeleton'; +import { LINK_DOCS_EXPERIMENTS_CLI } from '@studio/constants/links'; +import { + DELTA_COMPARISON_LABEL, + type RecentExperiment, +} from '@studio/routes/agents/AgentDetailRoute/overview/recentExperiments'; +import type { FC } from 'react'; + +interface RecentExperimentsPanelProps { + experiments: RecentExperiment[]; + isPending?: boolean; + /** Open an experiment's own route. Omitted for an experiment whose name never resolved. */ + onOpenExperiment: (experiment: RecentExperiment) => void; + /** Empty-state action: submit an evaluation for this agent. */ + onRunEvaluation?: () => void; +} + +/** + * The score itself is a bare float with no scale metadata, so it renders as-is rather than as a + * percentage — see {@link formatEvaluatorScore}. The delta is the opposite case: it is already a + * relative change (a ratio of two same-unit scores), so the percent sign is accurate whatever the + * underlying scale. One decimal keeps it to the width the tag has room for. + */ +const formatDelta = (delta: number): string => + `${delta > 0 ? '+' : delta < 0 ? '−' : ''}${Math.abs(delta).toFixed(1)}%`; + +/** + * How the agent is trending against each benchmark it is measured on, one card per experiment. + * + * Each card's pills are the evaluators that experiment reports, and the trendline is that + * evaluator's mean across the experiment's published evaluations. + */ +export const RecentExperimentsPanel: FC = ({ + experiments, + isPending, + onOpenExperiment, + onRunEvaluation, +}) => ( + + Recent experiments + + {isPending ? ( + + ) : experiments.length === 0 ? ( + + + + {'Review changes and compare multiple evaluation runs with experiments. '} + + Learn more + + . + + } + slotFooter={ + onRunEvaluation ? ( + + ) : null + } + /> + + + ) : ( + experiments.map((experiment) => ( + onOpenExperiment(experiment) : undefined} + /> + )) + )} + +); diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/recentExperiments.test.ts b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/recentExperiments.test.ts new file mode 100644 index 0000000000..1fa8d248cd --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/recentExperiments.test.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { toRecentExperiments } from '@studio/routes/agents/AgentDetailRoute/overview/recentExperiments'; +import type { AgentEvaluationRow } from '@studio/routes/agents/AgentDetailRoute/useAgentDetails'; + +interface EvalOptions { + experimentId?: string; + experimentName?: string | null; + experimentDescription?: string | null; + createdAt?: string; + scores?: Record; +} + +const evaluation = (name: string, options: EvalOptions = {}): AgentEvaluationRow => + ({ + id: `eval-${name}`, + name, + workspace: 'default', + experiment_ids: [options.experimentId ?? 'exp-1'], + dataset_name: 'dataset', + experimentName: options.experimentName ?? 'Primary use cases', + experimentDescription: options.experimentDescription ?? 'Every merge to main.', + created_at: options.createdAt ?? '2026-08-01T00:00:00Z', + aggregate_scores: Object.fromEntries( + Object.entries(options.scores ?? {}).map(([key, mean]) => [key, { mean }]) + ), + }) as AgentEvaluationRow; + +describe('toRecentExperiments', () => { + it('rolls evaluations up into one card per experiment, newest first', () => { + const result = toRecentExperiments([ + evaluation('b', { experimentId: 'exp-2', createdAt: '2026-08-10T00:00:00Z' }), + evaluation('a', { experimentId: 'exp-1', createdAt: '2026-08-05T00:00:00Z' }), + evaluation('a2', { experimentId: 'exp-1', createdAt: '2026-08-01T00:00:00Z' }), + ]); + + expect(result.map((row) => row.id)).toEqual(['exp-2', 'exp-1']); + expect(result[1]?.evaluationCount).toBe(2); + expect(result[0]?.description).toBe('Every merge to main.'); + }); + + it('orders each series oldest-first regardless of the order evaluations arrive in', () => { + const [experiment] = toRecentExperiments([ + evaluation('newest', { createdAt: '2026-08-10T00:00:00Z', scores: { solved: 0.6 } }), + evaluation('oldest', { createdAt: '2026-08-01T00:00:00Z', scores: { solved: 0.2 } }), + ]); + + expect(experiment?.series[0]?.points.map((point) => point.value)).toEqual([0.2, 0.6]); + expect(experiment?.series[0]?.value).toBe(0.6); + }); + + it('measures the delta against the newest score at least a week old, as a percent change', () => { + const [experiment] = toRecentExperiments([ + evaluation('latest', { createdAt: '2026-08-20T00:00:00Z', scores: { solved: 0.5 } }), + // Inside the window — must not be used as the baseline. + evaluation('recent', { createdAt: '2026-08-18T00:00:00Z', scores: { solved: 0.4 } }), + evaluation('week-ago', { createdAt: '2026-08-13T00:00:00Z', scores: { solved: 0.2 } }), + evaluation('older', { createdAt: '2026-08-01T00:00:00Z', scores: { solved: 0.1 } }), + ]); + + // .2 → .5 is a 150% increase, not a raw +0.3. + expect(experiment?.series[0]?.delta).toBeCloseTo(150); + }); + + it('reports a drop as a negative percent change', () => { + const [experiment] = toRecentExperiments([ + evaluation('latest', { createdAt: '2026-08-20T00:00:00Z', scores: { solved: 0.4 } }), + evaluation('baseline', { createdAt: '2026-08-01T00:00:00Z', scores: { solved: 0.5 } }), + ]); + + expect(experiment?.series[0]?.delta).toBeCloseTo(-20); + }); + + it('scales the percent change independently of the score magnitude', () => { + const asPercent = (from: number, to: number) => + toRecentExperiments([ + evaluation('latest', { createdAt: '2026-08-20T00:00:00Z', scores: { m: to } }), + evaluation('baseline', { createdAt: '2026-08-01T00:00:00Z', scores: { m: from } }), + ])[0]?.series[0]?.delta; + + // A 10% move reads the same whether the metric is a ratio, a point count, or a latency. + expect(asPercent(0.5, 0.55)).toBeCloseTo(10); + expect(asPercent(50, 55)).toBeCloseTo(10); + expect(asPercent(1200, 1320)).toBeCloseTo(10); + }); + + it('omits the delta when the baseline is zero, which has no percent change', () => { + const [experiment] = toRecentExperiments([ + evaluation('latest', { createdAt: '2026-08-20T00:00:00Z', scores: { solved: 0.4 } }), + evaluation('baseline', { createdAt: '2026-08-01T00:00:00Z', scores: { solved: 0 } }), + ]); + + expect(experiment?.series[0]?.delta).toBeUndefined(); + }); + + it('omits the delta when nothing is old enough to compare against', () => { + const [experiment] = toRecentExperiments([ + evaluation('latest', { createdAt: '2026-08-20T00:00:00Z', scores: { solved: 0.5 } }), + evaluation('recent', { createdAt: '2026-08-19T00:00:00Z', scores: { solved: 0.4 } }), + ]); + + expect(experiment?.series[0]?.delta).toBeUndefined(); + }); + + it('builds one alphabetized series per evaluator and humanizes the label', () => { + const [experiment] = toRecentExperiments([ + evaluation('a', { scores: { 'llm-judge.tool_use': 0.9, helpfulness: 0.5 } }), + ]); + + expect(experiment?.series.map((series) => series.label)).toEqual(['Helpfulness', 'Tool Use']); + }); + + it('drops evaluations with no timestamp from the series but still counts them', () => { + const [experiment] = toRecentExperiments([ + evaluation('dated', { createdAt: '2026-08-01T00:00:00Z', scores: { solved: 0.5 } }), + { ...evaluation('undated', { scores: { solved: 0.9 } }), created_at: undefined }, + ]); + + expect(experiment?.evaluationCount).toBe(2); + expect(experiment?.series[0]?.points).toHaveLength(1); + expect(experiment?.series[0]?.value).toBe(0.5); + }); + + it('ignores evaluations that belong to no experiment', () => { + expect(toRecentExperiments([{ ...evaluation('orphan'), experiment_ids: [] }])).toEqual([]); + }); + + it('caps the number of cards', () => { + const rows = ['exp-1', 'exp-2', 'exp-3', 'exp-4'].map((id, index) => + evaluation(id, { experimentId: id, createdAt: `2026-08-0${index + 1}T00:00:00Z` }) + ); + + expect(toRecentExperiments(rows)).toHaveLength(3); + }); +}); diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/recentExperiments.ts b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/recentExperiments.ts new file mode 100644 index 0000000000..6e214c0def --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/overview/recentExperiments.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { snakeCaseToTitleCase } from '@nemo/common/src/utils/formatters'; +import type { MetricTrendSeries } from '@studio/components/charts/MetricTrendPanel'; +import { evaluatorLabel } from '@studio/routes/agents/AgentDetailRoute/evaluations/formatRollups'; +import type { AgentEvaluationRow } from '@studio/routes/agents/AgentDetailRoute/useAgentDetails'; + +/** Enough to show what the agent is being measured against without turning the overview into the + * Evaluations tab. The rest stay one click away under Evaluations → Experiments. */ +export const RECENT_EXPERIMENT_LIMIT = 3; + +/** Window the delta is measured over. Fixed rather than "previous run" so the number means the same + * thing on an experiment that runs hourly and one that runs monthly. */ +export const DELTA_WINDOW_DAYS = 7; +export const DELTA_COMPARISON_LABEL = `vs. ${DELTA_WINDOW_DAYS} days ago`; + +const DELTA_WINDOW_MS = DELTA_WINDOW_DAYS * 24 * 60 * 60 * 1000; + +export interface RecentExperiment { + id: string; + /** Null when the experiment fell outside the fetched page and could not be resolved. Such a card + * keeps its scores but loses its label and its link. */ + name: string | null; + description: string | null; + latestCreatedAt: string | null; + evaluationCount: number; + /** One per evaluator seen anywhere in the experiment, alphabetized by label. */ + series: MetricTrendSeries[]; +} + +/** A score with the timestamp it was published at, which the delta needs and `MetricTrendPoint` drops. */ +interface StampedScore { + at: number; + value: number; +} + +const pointLabel = (at: number): string => + new Date(at).toLocaleDateString(undefined, { month: 'short', day: 'numeric' }); + +/** + * Change over the last {@link DELTA_WINDOW_DAYS}, as a percentage of the baseline: latest against + * the newest score that is at least a full window older. + * + * Relative rather than a raw difference because evaluator scores carry no scale metadata (see + * `formatEvaluatorScore`). A raw difference is unreadable at the scale scores actually land on — + * a solve rate moving .149 → .160 reads as "+0.011" — and it cannot be a percentage *point* either, + * since that would assume the score is a 0–1 ratio when it may be a point count or a latency. A + * ratio of two same-unit numbers is scale-free, so it stays meaningful whichever the score is. + * + * Undefined when nothing is old enough to compare against, or when the baseline is zero — there is + * no percentage change from nothing. Divided by the absolute baseline so a negative baseline still + * yields "moved up" for an increase. + */ +const deltaOverWindow = (scores: StampedScore[]): number | undefined => { + const latest = scores.at(-1); + if (!latest) return undefined; + const cutoff = latest.at - DELTA_WINDOW_MS; + const baseline = scores.filter((score) => score.at <= cutoff).at(-1); + if (!baseline || baseline.value === 0) return undefined; + return ((latest.value - baseline.value) / Math.abs(baseline.value)) * 100; +}; + +const toSeries = (evaluator: string, scores: StampedScore[]): MetricTrendSeries => ({ + id: evaluator, + label: snakeCaseToTitleCase(evaluatorLabel(evaluator)), + value: scores.at(-1)?.value ?? 0, + delta: deltaOverWindow(scores), + points: scores.map((score) => ({ label: pointLabel(score.at), value: score.value })), +}); + +/** + * Roll the agent's evaluations up into one trend card per experiment, newest first. + * + * Derived from the evaluations rather than queried, for the same reason as `groupByExperiment`: the + * experiments endpoint has no `agent_name` filter, so "which experiments cover this agent" is only + * answerable through the evaluations that name it. + * + * An evaluation with no `created_at` is dropped from the series — a trend line needs a position on + * the x-axis, and a point with no timestamp cannot be placed or compared. It still counts toward + * `evaluationCount` so the card does not under-report how much has run. + */ +export const toRecentExperiments = ( + evaluations: AgentEvaluationRow[], + limit: number = RECENT_EXPERIMENT_LIMIT +): RecentExperiment[] => { + const byExperiment = new Map< + string, + { row: RecentExperiment; scores: Map } + >(); + + for (const evaluation of evaluations) { + const id = evaluation.experiment_ids[0]; + if (!id) continue; + + const entry = byExperiment.get(id) ?? { + row: { + id, + name: evaluation.experimentName, + description: evaluation.experimentDescription, + latestCreatedAt: null, + evaluationCount: 0, + series: [], + }, + scores: new Map(), + }; + + entry.row.name ??= evaluation.experimentName; + entry.row.description ??= evaluation.experimentDescription; + entry.row.evaluationCount += 1; + if ( + evaluation.created_at && + (!entry.row.latestCreatedAt || evaluation.created_at > entry.row.latestCreatedAt) + ) { + entry.row.latestCreatedAt = evaluation.created_at; + } + + const at = Date.parse(evaluation.created_at ?? ''); + if (Number.isFinite(at)) { + for (const [evaluator, aggregate] of Object.entries(evaluation.aggregate_scores ?? {})) { + const mean = aggregate?.mean; + if (typeof mean !== 'number' || !Number.isFinite(mean)) continue; + const existing = entry.scores.get(evaluator) ?? []; + existing.push({ at, value: mean }); + entry.scores.set(evaluator, existing); + } + } + + byExperiment.set(id, entry); + } + + return [...byExperiment.values()] + .map(({ row, scores }) => ({ + ...row, + series: [...scores.entries()] + .map(([evaluator, values]) => + toSeries( + evaluator, + [...values].sort((a, b) => a.at - b.at) + ) + ) + .sort((a, b) => a.label.localeCompare(b.label)), + })) + .sort((a, b) => (b.latestCreatedAt ?? '').localeCompare(a.latestCreatedAt ?? '')) + .slice(0, limit); +}; diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/useAgentDetails.ts b/web/packages/studio/src/routes/agents/AgentDetailRoute/useAgentDetails.ts index 9b7d5635f2..506a63296e 100644 --- a/web/packages/studio/src/routes/agents/AgentDetailRoute/useAgentDetails.ts +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/useAgentDetails.ts @@ -6,7 +6,7 @@ import { useToast } from '@nemo/common/src/providers/toast/useToast'; import { getAgentsListDeploymentsQueryKey, useAgentsDeleteDeployment, - useAgentsListAgents, + useAgentsGetAgent, useAgentsListDeployments, } from '@nemo/sdk/generated/agents/api'; import { useListEvaluations, useListExperiments } from '@nemo/sdk/generated/platform/api'; @@ -23,8 +23,13 @@ const EXPERIMENT_PAGE_SIZE = 100; /** Statuses that will not change again, so polling can stop. */ const TERMINAL_JOB_STATUSES = new Set(['completed', 'error', 'cancelled']); -/** A published evaluation plus the experiment name its detail route is nested under. */ -export type AgentEvaluationRow = EvaluationResponse & { experimentName: string | null }; +/** A published evaluation plus the experiment it belongs to — the name its detail route is nested + * under, and the description the overview cards label it with. Both are null when the experiment + * falls outside the fetched page. */ +export type AgentEvaluationRow = EvaluationResponse & { + experimentName: string | null; + experimentDescription: string | null; +}; interface UseAgentPanelParams { workspace: string; @@ -40,8 +45,8 @@ export const useAgentDetails = ({ const queryClient = useQueryClient(); const toast = useToast(); - const { data: agentsResponse } = useAgentsListAgents(workspace, undefined, { - query: { enabled: !!agentName }, + const { data: agent } = useAgentsGetAgent(workspace, agentName ?? '', { + query: { enabled: !!agentName && !!workspace }, }); const { data: deploymentsResponse, isLoading: isDeploymentsLoading } = useAgentsListDeployments( @@ -66,7 +71,6 @@ export const useAgentDetails = ({ } ); - const agentsData = agentsResponse?.data; const deploymentsData = deploymentsResponse?.data; const { data: agentEvalsResponse } = useListEvaluations( @@ -107,7 +111,6 @@ export const useAgentDetails = ({ }, }); - const agent = agentName ? (agentsData ?? []).find((a) => a.name === agentName) : undefined; const agentDeployments = useMemo( () => (deploymentsData ?? []).filter((d) => d.agent === agentName), [deploymentsData, agentName] @@ -121,13 +124,17 @@ export const useAgentDetails = ({ const agentEvals: AgentEvaluationRow[] = useMemo(() => { if (!agentName) return []; - const namesById = new Map( - (experimentsResponse?.data ?? []).map((experiment) => [experiment.id, experiment.name]) + const byId = new Map( + (experimentsResponse?.data ?? []).map((experiment) => [experiment.id, experiment]) ); - return (agentEvalsResponse?.data ?? []).map((evaluation) => ({ - ...evaluation, - experimentName: namesById.get(evaluation.experiment_ids[0] ?? '') ?? null, - })); + return (agentEvalsResponse?.data ?? []).map((evaluation) => { + const experiment = byId.get(evaluation.experiment_ids[0] ?? ''); + return { + ...evaluation, + experimentName: experiment?.name ?? null, + experimentDescription: experiment?.description ?? null, + }; + }); }, [agentEvalsResponse, experimentsResponse, agentName]); const agentJobs: EvalJobRow[] = useMemo(() => {