From ba2c67d70b00568c12799b2faa426721008cd26c Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Thu, 20 Aug 2026 12:02:05 -0300 Subject: [PATCH] refactor(evaluator)!: make runner and agent-eval metrics built-in Storing a Gym or Harbor taskset meant bundling its reward metric with `CloudpickleMetricBundlePackager()` -- the opt-in the docs correctly frame as "shipping custom code" -- for a metric the platform owns. Same for the three agent-eval metrics. None of them is custom: their entire state is JSON-able scalars, so they belong in `MetricVariants` alongside the other 23 built-ins. Promotes five: `GymRewardMetric`, `HarborRewardMetric`, `AgentPhaseSuccessMetric`, `EvidencePresenceMetric`, `SkillUsedMetric`. Each gains a `MetricType` member and subclasses `MetricBase` directly, carrying its own discriminator and fields. No separate config class. The `values/` config + `metrics/` runtime split exists to keep heavy runtime deps out of the config layer -- `metrics/bleu.py` imports sacrebleu at module scope, so `values.metrics.BLEU` stays importable without it. These five add no deps over their config, every config class in that module has exactly one consumer (its own runtime subclass), and no values-level union requires them to be co-located, so the split would be indirection that buys nothing. The type strings are unchanged from what the metrics already emitted (`gym_reward`, `harbor_reward`, ...), so nothing moves on the wire; what changes is that they now bundle inline and rehydrate without executing pickled code. BREAKING: `type` is now a fixed discriminator, so the `metric_type` override is gone. `Field(discriminator="type")` cannot express a per-caller type string. The override was used in exactly one place repo-wide -- a test fixture -- and cost every caller the cloudpickle opt-in. The two reward metrics could not simply subclass `MetricBase` where they lived. `MetricBase` drags the dataset-schema stack (jinja2, jsonschema), and `harbor_runtime` is on the optimizer's light import path, guarded by `test_agent_eval_import_does_not_pull_the_execution_stack`. Defining them there turned that test red. They now live in `metrics/runner_rewards.py` on the heavy side, and `harbor_runtime` re-exports `HarborRewardMetric` through a module `__getattr__` with a `TYPE_CHECKING` declaration, so `from ...harbor_runtime import HarborRewardMetric` still works and still type-checks while the light path stays light. The two default-metric construction sites import locally for the same reason. `gym/results.py` gets no such shim. It declares no `__all__`, never published `GymRewardMetric` as part of its surface, and nothing imports the metric from that path -- the deep-path imports are all private helpers. The public path is `from ...runtimes.gym import GymRewardMetric`, which the package `__init__` serves from the canonical module. The skill's curated metric list deliberately does not gain these five. That page is about choosing a scorer for your data, and none of them is a choice -- they arrive with the runner or the harness. `test_skill_examples` records the reasoning next to the existing `tunable-rag-evaluator` exemption. Co-Authored-By: Claude Opus 5 Signed-off-by: Sandy Chapman --- .../nemo_evaluator_sdk/agent_eval/metrics.py | 82 ++++++++----------- .../agent_eval/runtimes/gym/__init__.py | 2 +- .../agent_eval/runtimes/gym/dataset.py | 10 ++- .../agent_eval/runtimes/gym/results.py | 28 ------- .../agent_eval/runtimes/harbor_runtime.py | 56 ++++++------- .../src/nemo_evaluator_sdk/enums.py | 9 ++ .../metrics/runner_rewards.py | 65 +++++++++++++++ .../src/nemo_evaluator_sdk/metrics/types.py | 11 +++ .../tests/agent_eval/test_metrics.py | 53 ++++++++++-- .../tests/test_skill_examples.py | 18 +++- .../beta/evaluator/agent_eval/metrics.py | 82 ++++++++----------- .../agent_eval/runtimes/gym/__init__.py | 2 +- .../agent_eval/runtimes/gym/dataset.py | 10 ++- .../agent_eval/runtimes/gym/results.py | 28 ------- .../agent_eval/runtimes/harbor_runtime.py | 56 ++++++------- .../src/nemo_platform/beta/evaluator/enums.py | 9 ++ .../beta/evaluator/metrics/runner_rewards.py | 65 +++++++++++++++ .../beta/evaluator/metrics/types.py | 11 +++ 18 files changed, 379 insertions(+), 218 deletions(-) create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/runner_rewards.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/runner_rewards.py diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/metrics.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/metrics.py index babc3b5e13..456edbb8b2 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/metrics.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/metrics.py @@ -20,9 +20,10 @@ import logging import math from collections.abc import Mapping -from typing import Any +from typing import Any, ClassVar, Literal from nemo_evaluator_sdk.agent_eval.trials import EVIDENCE_FINAL_STATE +from nemo_evaluator_sdk.enums import MetricType from nemo_evaluator_sdk.metrics.protocol import ( CandidateOutput, MetricInput, @@ -32,6 +33,7 @@ ) from nemo_evaluator_sdk.values.atif import Trajectory from nemo_evaluator_sdk.values.evidence import EVIDENCE_TRACE +from nemo_evaluator_sdk.values.metrics import MetricBase from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator logger = logging.getLogger(__name__) @@ -46,19 +48,18 @@ ) -class AgentPhaseSuccessMetric: +class AgentPhaseSuccessMetric(MetricBase): """Emit ``True`` when the agent phase exited successfully, else ``False``. - The metric ``type`` is overridable via the ``metric_type`` class attribute so - callers can namespace it; the output name stays ``agent_phase_success`` (which - gating reads as a reward signal — ``True``/``False`` coerces to ``1.0``/``0.0``). - """ + The output name stays ``agent_phase_success`` (which gating reads as a reward + signal — ``True``/``False`` coerces to ``1.0``/``0.0``). - metric_type: str = "agent_phase_success" + A built-in metric type, so it bundles inline and needs no cloudpickle opt-in to be + stored on a task. ``type`` is therefore a fixed discriminator and no longer + overridable per caller. + """ - @property - def type(self) -> str: - return self.metric_type + type: Literal[MetricType.AGENT_PHASE_SUCCESS] = MetricType.AGENT_PHASE_SUCCESS def output_spec(self) -> list[MetricOutputSpec]: return [MetricOutputSpec.boolean("agent_phase_success")] @@ -71,7 +72,7 @@ async def compute_scores(self, input: MetricInput) -> MetricResult: return MetricResult(outputs=[MetricOutput(name="agent_phase_success", value=agent_ok)]) -class EvidencePresenceMetric: +class EvidencePresenceMetric(MetricBase): """Emit ``True`` when a named filesystem evidence directory exists (and is non-empty). Reads ``candidate.evidence`` directly — the canonical metric-over-evidence @@ -79,43 +80,35 @@ class EvidencePresenceMetric: not a reward stamped into metadata by a verifier. """ - def __init__( - self, - *, - evidence_name: str = EVIDENCE_FINAL_STATE, - output_name: str = "evidence_present", - require_non_empty: bool = True, - ) -> None: - self._evidence_name = evidence_name - self._output_name = output_name - self._require_non_empty = require_non_empty - - @property - def type(self) -> str: - return "evidence_presence" + type: Literal[MetricType.EVIDENCE_PRESENCE] = MetricType.EVIDENCE_PRESENCE + evidence_name: str = Field(default=EVIDENCE_FINAL_STATE, description="Evidence directory to look for.") + output_name: str = Field(default="evidence_present", description="Name of the emitted boolean score.") + require_non_empty: bool = Field( + default=True, description="Require the evidence directory to be non-empty, not merely present." + ) def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.boolean(self._output_name)] + return [MetricOutputSpec.boolean(self.output_name)] async def compute_scores(self, input: MetricInput) -> MetricResult: present = False evidence = input.candidate.evidence - if evidence is not None and evidence.get(self._evidence_name) is not None: + if evidence is not None and evidence.get(self.evidence_name) is not None: try: - handle = await evidence.filesystem(self._evidence_name) + handle = await evidence.filesystem(self.evidence_name) if await handle.exists(): - present = bool(await handle.iter_paths(recursive=True)) if self._require_non_empty else True + present = bool(await handle.iter_paths(recursive=True)) if self.require_non_empty else True except (KeyError, ValueError) as exc: logger.warning( "EvidencePresenceMetric scored False: could not resolve evidence %r for output %r: %s", - self._evidence_name, - self._output_name, + self.evidence_name, + self.output_name, exc, ) - return MetricResult(outputs=[MetricOutput(name=self._output_name, value=present)]) + return MetricResult(outputs=[MetricOutput(name=self.output_name, value=present)]) -class SkillUsedMetric: +class SkillUsedMetric(MetricBase): """Emit ``skill_present`` and ``skill_used`` so an eval can flag a failure to use an injected skill. * ``skill_present`` — ``True`` when one or more skills were injected into the trial. Reads @@ -135,18 +128,13 @@ class SkillUsedMetric: With no skill present, both outputs are ``False``. """ - metric_type: str = "skill_used" - OUTPUT_PRESENT: str = "skill_present" - OUTPUT_USED: str = "skill_used" - # Metadata key skill-aware runtimes stamp the provenance list under (matches the fabric runtime). - _SKILLS_KEY: str = "skills" + type: Literal[MetricType.SKILL_USED] = MetricType.SKILL_USED + trace_evidence: str = Field(default=EVIDENCE_TRACE, description="Trace evidence to scan for skill usage.") - def __init__(self, *, trace_evidence: str = EVIDENCE_TRACE) -> None: - self._trace_evidence = trace_evidence - - @property - def type(self) -> str: - return self.metric_type + OUTPUT_PRESENT: ClassVar[str] = "skill_present" + OUTPUT_USED: ClassVar[str] = "skill_used" + # Metadata key skill-aware runtimes stamp the provenance list under (matches the fabric runtime). + _SKILLS_KEY: ClassVar[str] = "skills" def output_spec(self) -> list[MetricOutputSpec]: return [ @@ -176,15 +164,15 @@ async def _any_skill_used(self, candidate: CandidateOutput, provenances: list[Ma if not locations: return False evidence = candidate.evidence - if evidence is None or evidence.get(self._trace_evidence) is None: + if evidence is None or evidence.get(self.trace_evidence) is None: return False try: - trajectory = await (await evidence.trace(self._trace_evidence)).trace() + trajectory = await (await evidence.trace(self.trace_evidence)).trace() except (KeyError, ValueError, ValidationError, OSError) as exc: # Best-effort: a missing/malformed/invalid trajectory must score skill_used=False, not raise. # ValidationError covers Trajectory.model_validate; OSError covers the underlying file read. logger.warning( - "SkillUsedMetric scored skill_used=False: could not read trace %r: %s", self._trace_evidence, exc + "SkillUsedMetric scored skill_used=False: could not read trace %r: %s", self.trace_evidence, exc ) return False return any(_trajectory_references(trajectory, loc) for loc in locations) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py index 6e63a9fb06..61934debf1 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py @@ -72,8 +72,8 @@ from nemo_evaluator_sdk.agent_eval.runtimes.gym.config import DEFAULT_REWARD_KEY, GymRuntimeConfig from nemo_evaluator_sdk.agent_eval.runtimes.gym.dataset import discover_gym_tasks -from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import GymRewardMetric from nemo_evaluator_sdk.agent_eval.runtimes.gym.runtime import GymAgentTaskRunner +from nemo_evaluator_sdk.metrics.runner_rewards import GymRewardMetric __all__ = [ "DEFAULT_REWARD_KEY", diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py index 93c8e751dc..4890372450 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py @@ -22,7 +22,6 @@ NG_TASK_INDEX, _read_jsonl, ) -from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import GymRewardMetric from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask logger = logging.getLogger(__name__) @@ -94,6 +93,13 @@ def _render_instruction(responses_create_params: Mapping[str, Any]) -> str: return "\n\n".join(part for part in parts if part).strip() +def _default_gym_metric() -> object: + """The default reward metric, imported lazily (see ``metrics.runner_rewards``).""" + from nemo_evaluator_sdk.metrics.runner_rewards import GymRewardMetric + + return GymRewardMetric() + + def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = None) -> list[AgentEvalTask]: """Build one :class:`AgentEvalTask` per distinct row in a Gym dataset (jsonl). @@ -156,7 +162,7 @@ def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = N **({"instruction": instruction} if instruction else {}), "gym_row": params, }, - metrics=list(metrics) if metrics is not None else [GymRewardMetric()], + metrics=list(metrics) if metrics is not None else [_default_gym_metric()], metadata={ "gym_dataset_path": str(dataset), # Everything except responses_create_params, which already lives in inputs['gym_row']. diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py index 1f14e6f45e..33815f22b0 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py @@ -26,7 +26,6 @@ ) from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput -from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor from nemo_evaluator_sdk.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore @@ -43,33 +42,6 @@ #: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and #: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym #: *honors* a caller-supplied ``_ng_task_index``, which is what makes the join here deterministic. -class GymRewardMetric: - """Score the Gym verifier reward stamped onto trial metadata. - - The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward`` - off the candidate metadata (populated by :class:`GymAgentTaskRunner`); a trial - with no reward is left **unscored** (``None`` → ``nan``), excluded from the mean - and surfaced as ``nan_count`` rather than counted as a spurious ``0.0``. Gym owns - the scoring — this metric only surfaces it (Evaluator does not re-derive the reward). - """ - - def __init__(self, *, output_name: str = "reward", metric_type: str = "gym_reward") -> None: - self._output_name = output_name - self._metric_type = metric_type - - @property - def type(self) -> str: - return self._metric_type - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.continuous_score(self._output_name)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - reward = input.candidate.metadata.get("reward") - value = float(reward) if reward is not None else None - return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)]) - - def _agent_never_ran(record: Mapping[str, Any]) -> bool: """True when a result record shows the agent produced nothing *and* never called the model. diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py index a2bc6e0243..7be778b7cc 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py @@ -31,6 +31,11 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from nemo_evaluator_sdk.metrics.runner_rewards import HarborRewardMetric + import contextlib import hashlib import importlib.machinery @@ -60,7 +65,7 @@ TrialError, standard_evidence_descriptors, ) -from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values.evidence import ( EVIDENCE_FORMAT_ATIF, CandidateEvidence, @@ -178,32 +183,6 @@ def _agent_dir_needs_import_path(self) -> HarborRuntimeConfig: return self -class HarborRewardMetric: - """Score the verifier reward Harbor stamped onto trial metadata. - - Reads ``reward`` from the candidate metadata (populated by - :func:`build_trials_from_job_dir`); a trial with no verifier reward scores - ``0.0``. This is the Harbor analogue of the example ``VerifierRewardMetric`` - — a reward-off-metadata scorer. - """ - - def __init__(self, *, output_name: str = "reward", metric_type: str = "harbor_reward") -> None: - self._output_name = output_name - self._metric_type = metric_type - - @property - def type(self) -> str: - return self._metric_type - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.continuous_score(self._output_name)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - reward = input.candidate.metadata.get("reward") - value = float(reward) if reward is not None else 0.0 - return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)]) - - def _effective_harbor_agent(config: HarborRuntimeConfig | None) -> str | None: """The agent a run will actually use, mirroring ``run_job``'s resolution order. @@ -1373,7 +1352,7 @@ def discover_harbor_tasks(dataset_path: str | Path) -> list[AgentEvalTask]: # lives in `inputs["instruction"]`. intent=task_name, inputs={"instruction": instruction}, - metrics=[HarborRewardMetric()], + metrics=[_harbor_reward_metric()], metadata={"harbor_dataset_path": str(dataset_path), "harbor_task_dir": str(task_dir)}, ) ) @@ -1501,3 +1480,24 @@ def reward_payload_from_result( "run_harbor_eval", "scoped_harbor_agent_import", ] + + +def _harbor_reward_metric() -> "HarborRewardMetric": + """Build the default reward metric, importing it lazily to keep this module light.""" + from nemo_evaluator_sdk.metrics.runner_rewards import HarborRewardMetric + + return HarborRewardMetric() + + +def __getattr__(name: str) -> object: + """Re-export ``HarborRewardMetric`` without importing the metric stack at module scope. + + Defining it here would pull ``MetricBase`` and the dataset-schema machinery (jinja2, + jsonschema) onto the optimizer's light import path. See + ``nemo_evaluator_sdk.metrics.runner_rewards``. + """ + if name == "HarborRewardMetric": + from nemo_evaluator_sdk.metrics.runner_rewards import HarborRewardMetric + + return HarborRewardMetric + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/enums.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/enums.py index 26266fb0fc..583d02ca58 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/enums.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/enums.py @@ -36,6 +36,15 @@ class MetricType(str, Enum): NOISE_SENSITIVITY = "noise_sensitivity" TUNABLE_RAG_EVALUATOR = "tunable-rag-evaluator" + # Runner-owned rewards: the runner scores, these surface it. + GYM_REWARD = "gym_reward" + HARBOR_REWARD = "harbor_reward" + + # Agent-eval scoring over trial metadata and evidence. + AGENT_PHASE_SUCCESS = "agent_phase_success" + EVIDENCE_PRESENCE = "evidence_presence" + SKILL_USED = "skill_used" + SYSTEM = "system" diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/runner_rewards.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/runner_rewards.py new file mode 100644 index 0000000000..06fa2be7c8 --- /dev/null +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/runner_rewards.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime metrics that surface a runner's own reward. + +These live here rather than beside their runners because a built-in metric subclasses +``MetricBase``, which drags the dataset-schema stack (jinja2, jsonschema) with it. The Harbor +runtime is on the optimizer's light import path — see +``test_agent_eval_import_does_not_pull_the_execution_stack`` — so defining them there would make +every consumer of that module pay for machinery these metrics do not use. The runner modules +re-export them lazily, so ``from ...harbor_runtime import HarborRewardMetric`` still works. +""" + +from typing import Literal + +from nemo_evaluator_sdk.enums import MetricType +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_evaluator_sdk.values.metrics import MetricBase +from pydantic import Field + +__all__ = ["GymRewardMetric", "HarborRewardMetric"] + + +class GymRewardMetric(MetricBase): + """Score the Gym verifier reward stamped onto trial metadata. + + The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward`` off the + candidate metadata (populated by ``GymAgentTaskRunner``); a trial with no reward is left + **unscored** (``None`` → ``nan``), excluded from the mean and surfaced as ``nan_count`` rather + than counted as a spurious ``0.0``. Gym owns the scoring — this metric only surfaces it. + """ + + type: Literal[MetricType.GYM_REWARD] = MetricType.GYM_REWARD + output_name: str = Field( + default="reward", description="Name of the emitted score, read from the trial's `reward` metadata." + ) + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score(self.output_name)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + reward = input.candidate.metadata.get("reward") + value = float(reward) if reward is not None else None + return MetricResult(outputs=[MetricOutput(name=self.output_name, value=value)]) + + +class HarborRewardMetric(MetricBase): + """Score the verifier reward Harbor stamped onto trial metadata. + + Reads ``reward`` from the candidate metadata (populated by ``build_trials_from_job_dir``); a + trial with no verifier reward scores ``0.0``. + """ + + type: Literal[MetricType.HARBOR_REWARD] = MetricType.HARBOR_REWARD + output_name: str = Field( + default="reward", description="Name of the emitted score, read from the trial's `reward` metadata." + ) + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score(self.output_name)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + reward = input.candidate.metadata.get("reward") + value = float(reward) if reward is not None else 0.0 + return MetricResult(outputs=[MetricOutput(name=self.output_name, value=value)]) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/types.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/types.py index 3754032653..b8bb423b4e 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/types.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/types.py @@ -5,6 +5,11 @@ from typing import Annotated, TypeAlias +from nemo_evaluator_sdk.agent_eval.metrics import ( + AgentPhaseSuccessMetric, + EvidencePresenceMetric, + SkillUsedMetric, +) from nemo_evaluator_sdk.metrics.bleu import BLEUMetric from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.metrics.f1 import F1Metric @@ -26,6 +31,7 @@ ) from nemo_evaluator_sdk.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric from nemo_evaluator_sdk.metrics.rouge import ROUGEMetric +from nemo_evaluator_sdk.metrics.runner_rewards import GymRewardMetric, HarborRewardMetric from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric from nemo_evaluator_sdk.metrics.tool_calling import ToolCallingMetric from nemo_evaluator_sdk.metrics.tunable_rag_evaluator import TunableRagEvaluatorMetric @@ -55,6 +61,11 @@ | ResponseRelevancyMetric | FaithfulnessMetric | NoiseSensitivityMetric + | GymRewardMetric + | HarborRewardMetric + | AgentPhaseSuccessMetric + | EvidencePresenceMetric + | SkillUsedMetric ) """Raw union of SDK metric configuration models, excluding service-only system metrics.""" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_metrics.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_metrics.py index 453b7e079b..821ccfb8ce 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_metrics.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_metrics.py @@ -9,19 +9,30 @@ from pathlib import Path import pytest +from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric, unbundle_metric +from nemo_evaluator.shared.metric_bundles.inline import InlineMetricBundlePackager from nemo_evaluator_sdk.agent_eval.metrics import ( AgentPhaseSuccessMetric, EvidencePresenceMetric, + SkillUsedMetric, TrialMeasurements, ) from nemo_evaluator_sdk.agent_eval.trials import standard_evidence_descriptors from nemo_evaluator_sdk.metrics.protocol import CandidateOutput, DatasetRow, MetricInput +from nemo_evaluator_sdk.metrics.runner_rewards import GymRewardMetric, HarborRewardMetric from nemo_evaluator_sdk.values.evidence import CandidateEvidence from pydantic import ValidationError @pytest.mark.asyncio -async def test_agent_phase_success_metric_reads_metadata_and_namespaces_type() -> None: +async def test_agent_phase_success_metric_reads_metadata_and_bundles_inline() -> None: + """``type`` is a fixed discriminator now, which is what makes the metric inline-bundleable. + + It used to be overridable per subclass so callers could namespace it. That is incompatible with + ``MetricsUnion``'s ``Field(discriminator="type")``, and the override bought less than the + cloudpickle opt-in it cost: storing this metric on a task previously required shipping custom + code. + """ metric = AgentPhaseSuccessMetric() assert metric.type == "agent_phase_success" ok = await metric.compute_scores( @@ -29,10 +40,9 @@ async def test_agent_phase_success_metric_reads_metadata_and_namespaces_type() - ) assert ok.outputs[0].value is True - class Namespaced(AgentPhaseSuccessMetric): - metric_type = "agentic_use_agent_phase" - - assert Namespaced().type == "agentic_use_agent_phase" + bundle = bundle_metric(metric, InlineMetricBundlePackager()) + assert bundle.payload.kind == "inline", "a built-in metric must not need the cloudpickle opt-in" + assert type(unbundle_metric(bundle)) is AgentPhaseSuccessMetric @pytest.mark.asyncio @@ -122,3 +132,36 @@ def test_direct_construction_rejects_unserialisable_cost(bad: object) -> None: def test_direct_construction_still_accepts_a_finite_cost(good: object) -> None: # Rejecting non-finite values must not cost the ordinary coercion of a finite one. assert TrialMeasurements(cost_usd=good).cost_usd == float(good) + + +@pytest.mark.parametrize( + ("factory", "expected_type"), + [ + (lambda: GymRewardMetric(output_name="score"), "gym_reward"), + (lambda: HarborRewardMetric(output_name="score"), "harbor_reward"), + (AgentPhaseSuccessMetric, "agent_phase_success"), + (lambda: EvidencePresenceMetric(evidence_name="workspace", require_non_empty=False), "evidence_presence"), + (lambda: SkillUsedMetric(trace_evidence="atif"), "skill_used"), + ], +) +def test_runner_and_agent_eval_metrics_are_built_in(factory, expected_type: str) -> None: + """Each of these can be stored on a task without the cloudpickle opt-in. + + Non-default configuration is used deliberately: a metric that survives bundling only with its + defaults would still lose the caller's settings on the way to a stored task. + """ + metric = factory() + assert metric.type == expected_type + + bundle = bundle_metric(metric, InlineMetricBundlePackager()) + assert bundle.payload.kind == "inline" + + restored = unbundle_metric(bundle) + assert type(restored) is type(metric) + assert restored.model_dump() == metric.model_dump() + + +def test_built_in_metrics_reject_a_caller_supplied_type() -> None: + """The discriminator is fixed. Callers used to namespace it, which the union cannot express.""" + with pytest.raises(ValidationError): + GymRewardMetric(type="my_namespaced_reward") diff --git a/plugins/nemo-evaluator/tests/test_skill_examples.py b/plugins/nemo-evaluator/tests/test_skill_examples.py index c6c3a3888f..00f4173d73 100644 --- a/plugins/nemo-evaluator/tests/test_skill_examples.py +++ b/plugins/nemo-evaluator/tests/test_skill_examples.py @@ -420,11 +420,27 @@ def test_metric_selection_lists_exactly_the_supported_metric_names() -> None: `tunable-rag-evaluator` is registered for optimize / NAT-style judge flows but is intentionally omitted from this curated skill list until skill docs cover it. + + The runner and agent-eval metrics are omitted for a different reason: this page is about + *choosing a scorer for your data*, and none of them is a choice. They arrive with the runner + or the agent-eval harness -- `gym_reward` and `harbor_reward` surface a reward their runner + already computed, and the rest score trial metadata and evidence. They became registry members + so they bundle inline instead of demanding the cloudpickle opt-in, not so callers would pick + them off a list. """ from nemo_evaluator.cli import _is_ragas_metric, _metric_type_models # Registry metrics the skill may omit without failing this contract. - skill_omitted = frozenset({"tunable-rag-evaluator"}) + skill_omitted = frozenset( + { + "tunable-rag-evaluator", + "gym_reward", + "harbor_reward", + "agent_phase_success", + "evidence_presence", + "skill_used", + } + ) reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/metric-selection.md").read_text( encoding="utf-8" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py index 5c1171b90d..99af728cec 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/metrics.py @@ -20,9 +20,10 @@ import logging import math from collections.abc import Mapping -from typing import Any +from typing import Any, ClassVar, Literal from nemo_platform.beta.evaluator.agent_eval.trials import EVIDENCE_FINAL_STATE +from nemo_platform.beta.evaluator.enums import MetricType from nemo_platform.beta.evaluator.metrics.protocol import ( CandidateOutput, MetricInput, @@ -32,6 +33,7 @@ ) from nemo_platform.beta.evaluator.values.atif import Trajectory from nemo_platform.beta.evaluator.values.evidence import EVIDENCE_TRACE +from nemo_platform.beta.evaluator.values.metrics import MetricBase from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator logger = logging.getLogger(__name__) @@ -46,19 +48,18 @@ ) -class AgentPhaseSuccessMetric: +class AgentPhaseSuccessMetric(MetricBase): """Emit ``True`` when the agent phase exited successfully, else ``False``. - The metric ``type`` is overridable via the ``metric_type`` class attribute so - callers can namespace it; the output name stays ``agent_phase_success`` (which - gating reads as a reward signal — ``True``/``False`` coerces to ``1.0``/``0.0``). - """ + The output name stays ``agent_phase_success`` (which gating reads as a reward + signal — ``True``/``False`` coerces to ``1.0``/``0.0``). - metric_type: str = "agent_phase_success" + A built-in metric type, so it bundles inline and needs no cloudpickle opt-in to be + stored on a task. ``type`` is therefore a fixed discriminator and no longer + overridable per caller. + """ - @property - def type(self) -> str: - return self.metric_type + type: Literal[MetricType.AGENT_PHASE_SUCCESS] = MetricType.AGENT_PHASE_SUCCESS def output_spec(self) -> list[MetricOutputSpec]: return [MetricOutputSpec.boolean("agent_phase_success")] @@ -71,7 +72,7 @@ async def compute_scores(self, input: MetricInput) -> MetricResult: return MetricResult(outputs=[MetricOutput(name="agent_phase_success", value=agent_ok)]) -class EvidencePresenceMetric: +class EvidencePresenceMetric(MetricBase): """Emit ``True`` when a named filesystem evidence directory exists (and is non-empty). Reads ``candidate.evidence`` directly — the canonical metric-over-evidence @@ -79,43 +80,35 @@ class EvidencePresenceMetric: not a reward stamped into metadata by a verifier. """ - def __init__( - self, - *, - evidence_name: str = EVIDENCE_FINAL_STATE, - output_name: str = "evidence_present", - require_non_empty: bool = True, - ) -> None: - self._evidence_name = evidence_name - self._output_name = output_name - self._require_non_empty = require_non_empty - - @property - def type(self) -> str: - return "evidence_presence" + type: Literal[MetricType.EVIDENCE_PRESENCE] = MetricType.EVIDENCE_PRESENCE + evidence_name: str = Field(default=EVIDENCE_FINAL_STATE, description="Evidence directory to look for.") + output_name: str = Field(default="evidence_present", description="Name of the emitted boolean score.") + require_non_empty: bool = Field( + default=True, description="Require the evidence directory to be non-empty, not merely present." + ) def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.boolean(self._output_name)] + return [MetricOutputSpec.boolean(self.output_name)] async def compute_scores(self, input: MetricInput) -> MetricResult: present = False evidence = input.candidate.evidence - if evidence is not None and evidence.get(self._evidence_name) is not None: + if evidence is not None and evidence.get(self.evidence_name) is not None: try: - handle = await evidence.filesystem(self._evidence_name) + handle = await evidence.filesystem(self.evidence_name) if await handle.exists(): - present = bool(await handle.iter_paths(recursive=True)) if self._require_non_empty else True + present = bool(await handle.iter_paths(recursive=True)) if self.require_non_empty else True except (KeyError, ValueError) as exc: logger.warning( "EvidencePresenceMetric scored False: could not resolve evidence %r for output %r: %s", - self._evidence_name, - self._output_name, + self.evidence_name, + self.output_name, exc, ) - return MetricResult(outputs=[MetricOutput(name=self._output_name, value=present)]) + return MetricResult(outputs=[MetricOutput(name=self.output_name, value=present)]) -class SkillUsedMetric: +class SkillUsedMetric(MetricBase): """Emit ``skill_present`` and ``skill_used`` so an eval can flag a failure to use an injected skill. * ``skill_present`` — ``True`` when one or more skills were injected into the trial. Reads @@ -135,18 +128,13 @@ class SkillUsedMetric: With no skill present, both outputs are ``False``. """ - metric_type: str = "skill_used" - OUTPUT_PRESENT: str = "skill_present" - OUTPUT_USED: str = "skill_used" - # Metadata key skill-aware runtimes stamp the provenance list under (matches the fabric runtime). - _SKILLS_KEY: str = "skills" + type: Literal[MetricType.SKILL_USED] = MetricType.SKILL_USED + trace_evidence: str = Field(default=EVIDENCE_TRACE, description="Trace evidence to scan for skill usage.") - def __init__(self, *, trace_evidence: str = EVIDENCE_TRACE) -> None: - self._trace_evidence = trace_evidence - - @property - def type(self) -> str: - return self.metric_type + OUTPUT_PRESENT: ClassVar[str] = "skill_present" + OUTPUT_USED: ClassVar[str] = "skill_used" + # Metadata key skill-aware runtimes stamp the provenance list under (matches the fabric runtime). + _SKILLS_KEY: ClassVar[str] = "skills" def output_spec(self) -> list[MetricOutputSpec]: return [ @@ -176,15 +164,15 @@ async def _any_skill_used(self, candidate: CandidateOutput, provenances: list[Ma if not locations: return False evidence = candidate.evidence - if evidence is None or evidence.get(self._trace_evidence) is None: + if evidence is None or evidence.get(self.trace_evidence) is None: return False try: - trajectory = await (await evidence.trace(self._trace_evidence)).trace() + trajectory = await (await evidence.trace(self.trace_evidence)).trace() except (KeyError, ValueError, ValidationError, OSError) as exc: # Best-effort: a missing/malformed/invalid trajectory must score skill_used=False, not raise. # ValidationError covers Trajectory.model_validate; OSError covers the underlying file read. logger.warning( - "SkillUsedMetric scored skill_used=False: could not read trace %r: %s", self._trace_evidence, exc + "SkillUsedMetric scored skill_used=False: could not read trace %r: %s", self.trace_evidence, exc ) return False return any(_trajectory_references(trajectory, loc) for loc in locations) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py index 3ff68a9f8a..499d431d56 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py @@ -72,8 +72,8 @@ from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.config import DEFAULT_REWARD_KEY, GymRuntimeConfig from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.dataset import discover_gym_tasks -from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.results import GymRewardMetric from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.runtime import GymAgentTaskRunner +from nemo_platform.beta.evaluator.metrics.runner_rewards import GymRewardMetric __all__ = [ "DEFAULT_REWARD_KEY", diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/dataset.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/dataset.py index 24a53a35e9..7a1418a6c0 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/dataset.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/dataset.py @@ -22,7 +22,6 @@ NG_TASK_INDEX, _read_jsonl, ) -from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.results import GymRewardMetric from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask logger = logging.getLogger(__name__) @@ -94,6 +93,13 @@ def _render_instruction(responses_create_params: Mapping[str, Any]) -> str: return "\n\n".join(part for part in parts if part).strip() +def _default_gym_metric() -> object: + """The default reward metric, imported lazily (see ``metrics.runner_rewards``).""" + from nemo_platform.beta.evaluator.metrics.runner_rewards import GymRewardMetric + + return GymRewardMetric() + + def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = None) -> list[AgentEvalTask]: """Build one :class:`AgentEvalTask` per distinct row in a Gym dataset (jsonl). @@ -156,7 +162,7 @@ def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = N **({"instruction": instruction} if instruction else {}), "gym_row": params, }, - metrics=list(metrics) if metrics is not None else [GymRewardMetric()], + metrics=list(metrics) if metrics is not None else [_default_gym_metric()], metadata={ "gym_dataset_path": str(dataset), # Everything except responses_create_params, which already lives in inputs['gym_row']. diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/results.py index 6c5c9c09ff..73558517e7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/results.py @@ -26,7 +26,6 @@ ) from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput -from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor from nemo_platform.beta.evaluator.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore @@ -43,33 +42,6 @@ #: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and #: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym #: *honors* a caller-supplied ``_ng_task_index``, which is what makes the join here deterministic. -class GymRewardMetric: - """Score the Gym verifier reward stamped onto trial metadata. - - The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward`` - off the candidate metadata (populated by :class:`GymAgentTaskRunner`); a trial - with no reward is left **unscored** (``None`` → ``nan``), excluded from the mean - and surfaced as ``nan_count`` rather than counted as a spurious ``0.0``. Gym owns - the scoring — this metric only surfaces it (Evaluator does not re-derive the reward). - """ - - def __init__(self, *, output_name: str = "reward", metric_type: str = "gym_reward") -> None: - self._output_name = output_name - self._metric_type = metric_type - - @property - def type(self) -> str: - return self._metric_type - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.continuous_score(self._output_name)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - reward = input.candidate.metadata.get("reward") - value = float(reward) if reward is not None else None - return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)]) - - def _agent_never_ran(record: Mapping[str, Any]) -> bool: """True when a result record shows the agent produced nothing *and* never called the model. diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py index cdee910bfd..797baab90c 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py @@ -31,6 +31,11 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from nemo_platform.beta.evaluator.metrics.runner_rewards import HarborRewardMetric + import contextlib import hashlib import importlib.machinery @@ -60,7 +65,7 @@ TrialError, standard_evidence_descriptors, ) -from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_platform.beta.evaluator.metrics.protocol import Metric from nemo_platform.beta.evaluator.values.evidence import ( EVIDENCE_FORMAT_ATIF, CandidateEvidence, @@ -178,32 +183,6 @@ def _agent_dir_needs_import_path(self) -> HarborRuntimeConfig: return self -class HarborRewardMetric: - """Score the verifier reward Harbor stamped onto trial metadata. - - Reads ``reward`` from the candidate metadata (populated by - :func:`build_trials_from_job_dir`); a trial with no verifier reward scores - ``0.0``. This is the Harbor analogue of the example ``VerifierRewardMetric`` - — a reward-off-metadata scorer. - """ - - def __init__(self, *, output_name: str = "reward", metric_type: str = "harbor_reward") -> None: - self._output_name = output_name - self._metric_type = metric_type - - @property - def type(self) -> str: - return self._metric_type - - def output_spec(self) -> list[MetricOutputSpec]: - return [MetricOutputSpec.continuous_score(self._output_name)] - - async def compute_scores(self, input: MetricInput) -> MetricResult: - reward = input.candidate.metadata.get("reward") - value = float(reward) if reward is not None else 0.0 - return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)]) - - def _effective_harbor_agent(config: HarborRuntimeConfig | None) -> str | None: """The agent a run will actually use, mirroring ``run_job``'s resolution order. @@ -1373,7 +1352,7 @@ def discover_harbor_tasks(dataset_path: str | Path) -> list[AgentEvalTask]: # lives in `inputs["instruction"]`. intent=task_name, inputs={"instruction": instruction}, - metrics=[HarborRewardMetric()], + metrics=[_harbor_reward_metric()], metadata={"harbor_dataset_path": str(dataset_path), "harbor_task_dir": str(task_dir)}, ) ) @@ -1501,3 +1480,24 @@ def reward_payload_from_result( "run_harbor_eval", "scoped_harbor_agent_import", ] + + +def _harbor_reward_metric() -> "HarborRewardMetric": + """Build the default reward metric, importing it lazily to keep this module light.""" + from nemo_platform.beta.evaluator.metrics.runner_rewards import HarborRewardMetric + + return HarborRewardMetric() + + +def __getattr__(name: str) -> object: + """Re-export ``HarborRewardMetric`` without importing the metric stack at module scope. + + Defining it here would pull ``MetricBase`` and the dataset-schema machinery (jinja2, + jsonschema) onto the optimizer's light import path. See + ``nemo_platform.beta.evaluator.metrics.runner_rewards``. + """ + if name == "HarborRewardMetric": + from nemo_platform.beta.evaluator.metrics.runner_rewards import HarborRewardMetric + + return HarborRewardMetric + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/enums.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/enums.py index 26266fb0fc..583d02ca58 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/enums.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/enums.py @@ -36,6 +36,15 @@ class MetricType(str, Enum): NOISE_SENSITIVITY = "noise_sensitivity" TUNABLE_RAG_EVALUATOR = "tunable-rag-evaluator" + # Runner-owned rewards: the runner scores, these surface it. + GYM_REWARD = "gym_reward" + HARBOR_REWARD = "harbor_reward" + + # Agent-eval scoring over trial metadata and evidence. + AGENT_PHASE_SUCCESS = "agent_phase_success" + EVIDENCE_PRESENCE = "evidence_presence" + SKILL_USED = "skill_used" + SYSTEM = "system" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/runner_rewards.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/runner_rewards.py new file mode 100644 index 0000000000..76759970fa --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/runner_rewards.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime metrics that surface a runner's own reward. + +These live here rather than beside their runners because a built-in metric subclasses +``MetricBase``, which drags the dataset-schema stack (jinja2, jsonschema) with it. The Harbor +runtime is on the optimizer's light import path — see +``test_agent_eval_import_does_not_pull_the_execution_stack`` — so defining them there would make +every consumer of that module pay for machinery these metrics do not use. The runner modules +re-export them lazily, so ``from ...harbor_runtime import HarborRewardMetric`` still works. +""" + +from typing import Literal + +from nemo_platform.beta.evaluator.enums import MetricType +from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_platform.beta.evaluator.values.metrics import MetricBase +from pydantic import Field + +__all__ = ["GymRewardMetric", "HarborRewardMetric"] + + +class GymRewardMetric(MetricBase): + """Score the Gym verifier reward stamped onto trial metadata. + + The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward`` off the + candidate metadata (populated by ``GymAgentTaskRunner``); a trial with no reward is left + **unscored** (``None`` → ``nan``), excluded from the mean and surfaced as ``nan_count`` rather + than counted as a spurious ``0.0``. Gym owns the scoring — this metric only surfaces it. + """ + + type: Literal[MetricType.GYM_REWARD] = MetricType.GYM_REWARD + output_name: str = Field( + default="reward", description="Name of the emitted score, read from the trial's `reward` metadata." + ) + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score(self.output_name)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + reward = input.candidate.metadata.get("reward") + value = float(reward) if reward is not None else None + return MetricResult(outputs=[MetricOutput(name=self.output_name, value=value)]) + + +class HarborRewardMetric(MetricBase): + """Score the verifier reward Harbor stamped onto trial metadata. + + Reads ``reward`` from the candidate metadata (populated by ``build_trials_from_job_dir``); a + trial with no verifier reward scores ``0.0``. + """ + + type: Literal[MetricType.HARBOR_REWARD] = MetricType.HARBOR_REWARD + output_name: str = Field( + default="reward", description="Name of the emitted score, read from the trial's `reward` metadata." + ) + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score(self.output_name)] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + reward = input.candidate.metadata.get("reward") + value = float(reward) if reward is not None else 0.0 + return MetricResult(outputs=[MetricOutput(name=self.output_name, value=value)]) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/types.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/types.py index 48e79d61ee..701969b2d7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/types.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/types.py @@ -5,6 +5,11 @@ from typing import Annotated, TypeAlias +from nemo_platform.beta.evaluator.agent_eval.metrics import ( + AgentPhaseSuccessMetric, + EvidencePresenceMetric, + SkillUsedMetric, +) from nemo_platform.beta.evaluator.metrics.bleu import BLEUMetric from nemo_platform.beta.evaluator.metrics.exact_match import ExactMatchMetric from nemo_platform.beta.evaluator.metrics.f1 import F1Metric @@ -26,6 +31,7 @@ ) from nemo_platform.beta.evaluator.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric from nemo_platform.beta.evaluator.metrics.rouge import ROUGEMetric +from nemo_platform.beta.evaluator.metrics.runner_rewards import GymRewardMetric, HarborRewardMetric from nemo_platform.beta.evaluator.metrics.string_check import StringCheckMetric from nemo_platform.beta.evaluator.metrics.tool_calling import ToolCallingMetric from nemo_platform.beta.evaluator.metrics.tunable_rag_evaluator import TunableRagEvaluatorMetric @@ -55,6 +61,11 @@ | ResponseRelevancyMetric | FaithfulnessMetric | NoiseSensitivityMetric + | GymRewardMetric + | HarborRewardMetric + | AgentPhaseSuccessMetric + | EvidencePresenceMetric + | SkillUsedMetric ) """Raw union of SDK metric configuration models, excluding service-only system metrics."""