refactor(evaluator)!: make runner and agent-eval metrics built-in

Storing a Gym or Harbor taskset meant bundling its reward metric with
`CloudpickleMetricBundlePackager()` -- the opt-in the docs correctly frame as
"shipping custom code" -- for a metric the platform owns. Same for the three
agent-eval metrics. None of them is custom: their entire state is JSON-able
scalars, so they belong in `MetricVariants` alongside the other 23 built-ins.

Promotes five: `GymRewardMetric`, `HarborRewardMetric`, `AgentPhaseSuccessMetric`,
`EvidencePresenceMetric`, `SkillUsedMetric`. Each gains a `MetricType` member and
subclasses `MetricBase` directly, carrying its own discriminator and fields.

No separate config class. The `values/` config + `metrics/` runtime split exists
to keep heavy runtime deps out of the config layer -- `metrics/bleu.py` imports
sacrebleu at module scope, so `values.metrics.BLEU` stays importable without it.
These five add no deps over their config, every config class in that module has
exactly one consumer (its own runtime subclass), and no values-level union
requires them to be co-located, so the split would be indirection that buys
nothing. The type strings are unchanged from what the metrics already emitted
(`gym_reward`, `harbor_reward`, ...), so nothing moves on the wire; what changes
is that they now bundle inline and rehydrate without executing pickled code.

BREAKING: `type` is now a fixed discriminator, so the `metric_type` override is
gone. `Field(discriminator="type")` cannot express a per-caller type string. The
override was used in exactly one place repo-wide -- a test fixture -- and cost
every caller the cloudpickle opt-in.

The two reward metrics could not simply subclass `MetricBase` where they lived.
`MetricBase` drags the dataset-schema stack (jinja2, jsonschema), and
`harbor_runtime` is on the optimizer's light import path, guarded by
`test_agent_eval_import_does_not_pull_the_execution_stack`. Defining them there
turned that test red. They now live in `metrics/runner_rewards.py` on the heavy
side, and `harbor_runtime` re-exports `HarborRewardMetric` through a module
`__getattr__` with a `TYPE_CHECKING` declaration, so `from ...harbor_runtime
import HarborRewardMetric` still works and still type-checks while the light path
stays light. The two default-metric construction sites import locally for the
same reason.

`gym/results.py` gets no such shim. It declares no `__all__`, never published
`GymRewardMetric` as part of its surface, and nothing imports the metric from
that path -- the deep-path imports are all private helpers. The public path is
`from ...runtimes.gym import GymRewardMetric`, which the package `__init__`
serves from the canonical module.

The skill's curated metric list deliberately does not gain these five. That page
is about choosing a scorer for your data, and none of them is a choice -- they
arrive with the runner or the harness. `test_skill_examples` records the
reasoning next to the existing `tunable-rag-evaluator` exemption.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Signed-off-by: Sandy Chapman <schapman@nvidia.com>

Author: SandyChapman

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/metrics.py

@@ -19,9 +19,10 @@
 import json
 import logging
 from collections.abc import Mapping
-from typing import Any
+from typing import Any, ClassVar, Literal
 
-from nemo_evaluator_sdk.agent_eval.trials import EVIDENCE_FINAL_STATE
+from nemo_evaluator_sdk.agent_eval.trials import EVIDENCE_FINAL_STATE, EVIDENCE_TRACE
+from nemo_evaluator_sdk.enums import MetricType
 from nemo_evaluator_sdk.metrics.protocol import (
     CandidateOutput,
     MetricInput,
@@ -30,8 +31,8 @@
     MetricResult,
 )
 from nemo_evaluator_sdk.values.atif import Trajectory
-from nemo_evaluator_sdk.values.evidence import EVIDENCE_TRACE
-from pydantic import BaseModel, ConfigDict, ValidationError
+from nemo_evaluator_sdk.values.metrics import MetricBase
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
 
 logger = logging.getLogger(__name__)
 
@@ -45,19 +46,18 @@
 )
 
 
-class AgentPhaseSuccessMetric:
+class AgentPhaseSuccessMetric(MetricBase):
     """Emit ``True`` when the agent phase exited successfully, else ``False``.
 
-    The metric ``type`` is overridable via the ``metric_type`` class attribute so
-    callers can namespace it; the output name stays ``agent_phase_success`` (which
-    gating reads as a reward signal — ``True``/``False`` coerces to ``1.0``/``0.0``).
-    """
+    The output name stays ``agent_phase_success`` (which gating reads as a reward
+    signal — ``True``/``False`` coerces to ``1.0``/``0.0``).
 
-    metric_type: str = "agent_phase_success"
+    A built-in metric type, so it bundles inline and needs no cloudpickle opt-in to be
+    stored on a task. ``type`` is therefore a fixed discriminator and no longer
+    overridable per caller.
+    """
 
-    @property
-    def type(self) -> str:
-        return self.metric_type
+    type: Literal[MetricType.AGENT_PHASE_SUCCESS] = MetricType.AGENT_PHASE_SUCCESS
 
     def output_spec(self) -> list[MetricOutputSpec]:
         return [MetricOutputSpec.boolean("agent_phase_success")]
@@ -70,51 +70,43 @@ async def compute_scores(self, input: MetricInput) -> MetricResult:
         return MetricResult(outputs=[MetricOutput(name="agent_phase_success", value=agent_ok)])
 
 
-class EvidencePresenceMetric:
+class EvidencePresenceMetric(MetricBase):
     """Emit ``True`` when a named filesystem evidence directory exists (and is non-empty).
 
     Reads ``candidate.evidence`` directly — the canonical metric-over-evidence
     pattern — so the result reflects what the agent actually produced on disk,
     not a reward stamped into metadata by a verifier.
     """
 
-    def __init__(
-        self,
-        *,
-        evidence_name: str = EVIDENCE_FINAL_STATE,
-        output_name: str = "evidence_present",
-        require_non_empty: bool = True,
-    ) -> None:
-        self._evidence_name = evidence_name
-        self._output_name = output_name
-        self._require_non_empty = require_non_empty
-
-    @property
-    def type(self) -> str:
-        return "evidence_presence"
+    type: Literal[MetricType.EVIDENCE_PRESENCE] = MetricType.EVIDENCE_PRESENCE
+    evidence_name: str = Field(default=EVIDENCE_FINAL_STATE, description="Evidence directory to look for.")
+    output_name: str = Field(default="evidence_present", description="Name of the emitted boolean score.")
+    require_non_empty: bool = Field(
+        default=True, description="Require the evidence directory to be non-empty, not merely present."
+    )
 
     def output_spec(self) -> list[MetricOutputSpec]:
-        return [MetricOutputSpec.boolean(self._output_name)]
+        return [MetricOutputSpec.boolean(self.output_name)]
 
     async def compute_scores(self, input: MetricInput) -> MetricResult:
         present = False
         evidence = input.candidate.evidence
-        if evidence is not None and evidence.get(self._evidence_name) is not None:
+        if evidence is not None and evidence.get(self.evidence_name) is not None:
             try:
-                handle = await evidence.filesystem(self._evidence_name)
+                handle = await evidence.filesystem(self.evidence_name)
                 if await handle.exists():
-                    present = bool(await handle.iter_paths(recursive=True)) if self._require_non_empty else True
+                    present = bool(await handle.iter_paths(recursive=True)) if self.require_non_empty else True
             except (KeyError, ValueError) as exc:
                 logger.warning(
                     "EvidencePresenceMetric scored False: could not resolve evidence %r for output %r: %s",
-                    self._evidence_name,
-                    self._output_name,
+                    self.evidence_name,
+                    self.output_name,
                     exc,
                 )
-        return MetricResult(outputs=[MetricOutput(name=self._output_name, value=present)])
+        return MetricResult(outputs=[MetricOutput(name=self.output_name, value=present)])
 
 
-class SkillUsedMetric:
+class SkillUsedMetric(MetricBase):
     """Emit ``skill_present`` and ``skill_used`` so an eval can flag a failure to use an injected skill.
 
     * ``skill_present`` — ``True`` when one or more skills were injected into the trial. Reads
@@ -134,18 +126,13 @@ class SkillUsedMetric:
     With no skill present, both outputs are ``False``.
     """
 
-    metric_type: str = "skill_used"
-    OUTPUT_PRESENT: str = "skill_present"
-    OUTPUT_USED: str = "skill_used"
-    # Metadata key skill-aware runtimes stamp the provenance list under (matches the fabric runtime).
-    _SKILLS_KEY: str = "skills"
+    type: Literal[MetricType.SKILL_USED] = MetricType.SKILL_USED
+    trace_evidence: str = Field(default=EVIDENCE_TRACE, description="Trace evidence to scan for skill usage.")
 
-    def __init__(self, *, trace_evidence: str = EVIDENCE_TRACE) -> None:
-        self._trace_evidence = trace_evidence
-
-    @property
-    def type(self) -> str:
-        return self.metric_type
+    OUTPUT_PRESENT: ClassVar[str] = "skill_present"
+    OUTPUT_USED: ClassVar[str] = "skill_used"
+    # Metadata key skill-aware runtimes stamp the provenance list under (matches the fabric runtime).
+    _SKILLS_KEY: ClassVar[str] = "skills"
 
     def output_spec(self) -> list[MetricOutputSpec]:
         return [
@@ -175,15 +162,15 @@ async def _any_skill_used(self, candidate: CandidateOutput, provenances: list[Ma
         if not locations:
             return False
         evidence = candidate.evidence
-        if evidence is None or evidence.get(self._trace_evidence) is None:
+        if evidence is None or evidence.get(self.trace_evidence) is None:
             return False
         try:
-            trajectory = await (await evidence.trace(self._trace_evidence)).trace()
+            trajectory = await (await evidence.trace(self.trace_evidence)).trace()
         except (KeyError, ValueError, ValidationError, OSError) as exc:
             # Best-effort: a missing/malformed/invalid trajectory must score skill_used=False, not raise.
             # ValidationError covers Trajectory.model_validate; OSError covers the underlying file read.
             logger.warning(
-                "SkillUsedMetric scored skill_used=False: could not read trace %r: %s", self._trace_evidence, exc
+                "SkillUsedMetric scored skill_used=False: could not read trace %r: %s", self.trace_evidence, exc
             )
             return False
         return any(_trajectory_references(trajectory, loc) for loc in locations)

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py

@@ -72,8 +72,8 @@
 
 from nemo_evaluator_sdk.agent_eval.runtimes.gym.config import DEFAULT_REWARD_KEY, GymRuntimeConfig
 from nemo_evaluator_sdk.agent_eval.runtimes.gym.dataset import discover_gym_tasks
-from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import GymRewardMetric
 from nemo_evaluator_sdk.agent_eval.runtimes.gym.runtime import GymAgentTaskRunner
+from nemo_evaluator_sdk.metrics.runner_rewards import GymRewardMetric
 
 __all__ = [
     "DEFAULT_REWARD_KEY",

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py

@@ -22,7 +22,6 @@
     NG_TASK_INDEX,
     _read_jsonl,
 )
-from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import GymRewardMetric
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask
 
 logger = logging.getLogger(__name__)
@@ -94,6 +93,13 @@ def _render_instruction(responses_create_params: Mapping[str, Any]) -> str:
     return "\n\n".join(part for part in parts if part).strip()
 
 
+def _default_gym_metric() -> object:
+    """The default reward metric, imported lazily (see ``metrics.runner_rewards``)."""
+    from nemo_evaluator_sdk.metrics.runner_rewards import GymRewardMetric
+
+    return GymRewardMetric()
+
+
 def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = None) -> list[AgentEvalTask]:
     """Build one :class:`AgentEvalTask` per distinct row in a Gym dataset (jsonl).
 
@@ -156,7 +162,7 @@ def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = N
                     **({"instruction": instruction} if instruction else {}),
                     "gym_row": params,
                 },
-                metrics=list(metrics) if metrics is not None else [GymRewardMetric()],
+                metrics=list(metrics) if metrics is not None else [_default_gym_metric()],
                 metadata={
                     "gym_dataset_path": str(dataset),
                     # Everything except responses_create_params, which already lives in inputs['gym_row'].

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py

@@ -26,7 +26,6 @@
 )
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask
 from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput
-from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
 from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
 from nemo_evaluator_sdk.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore
 
@@ -43,33 +42,6 @@
 #: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and
 #: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym
 #: *honors* a caller-supplied ``_ng_task_index``, which is what makes the join here deterministic.
-class GymRewardMetric:
-    """Score the Gym verifier reward stamped onto trial metadata.
-
-    The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward``
-    off the candidate metadata (populated by :class:`GymAgentTaskRunner`); a trial
-    with no reward is left **unscored** (``None`` → ``nan``), excluded from the mean
-    and surfaced as ``nan_count`` rather than counted as a spurious ``0.0``. Gym owns
-    the scoring — this metric only surfaces it (Evaluator does not re-derive the reward).
-    """
-
-    def __init__(self, *, output_name: str = "reward", metric_type: str = "gym_reward") -> None:
-        self._output_name = output_name
-        self._metric_type = metric_type
-
-    @property
-    def type(self) -> str:
-        return self._metric_type
-
-    def output_spec(self) -> list[MetricOutputSpec]:
-        return [MetricOutputSpec.continuous_score(self._output_name)]
-
-    async def compute_scores(self, input: MetricInput) -> MetricResult:
-        reward = input.candidate.metadata.get("reward")
-        value = float(reward) if reward is not None else None
-        return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)])
-
-
 def _agent_never_ran(record: Mapping[str, Any]) -> bool:
     """True when a result record shows the agent produced nothing *and* never called the model.
 

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py

@@ -31,6 +31,11 @@
 
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from nemo_evaluator_sdk.metrics.runner_rewards import HarborRewardMetric
+
 import contextlib
 import hashlib
 import importlib.machinery
@@ -60,7 +65,7 @@
     TrialError,
     standard_evidence_descriptors,
 )
-from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_evaluator_sdk.metrics.protocol import Metric
 from nemo_evaluator_sdk.values.evidence import CandidateEvidence
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
@@ -174,32 +179,6 @@ def _agent_dir_needs_import_path(self) -> HarborRuntimeConfig:
         return self
 
 
-class HarborRewardMetric:
-    """Score the verifier reward Harbor stamped onto trial metadata.
-
-    Reads ``reward`` from the candidate metadata (populated by
-    :func:`build_trials_from_job_dir`); a trial with no verifier reward scores
-    ``0.0``. This is the Harbor analogue of the example ``VerifierRewardMetric``
-    — a reward-off-metadata scorer.
-    """
-
-    def __init__(self, *, output_name: str = "reward", metric_type: str = "harbor_reward") -> None:
-        self._output_name = output_name
-        self._metric_type = metric_type
-
-    @property
-    def type(self) -> str:
-        return self._metric_type
-
-    def output_spec(self) -> list[MetricOutputSpec]:
-        return [MetricOutputSpec.continuous_score(self._output_name)]
-
-    async def compute_scores(self, input: MetricInput) -> MetricResult:
-        reward = input.candidate.metadata.get("reward")
-        value = float(reward) if reward is not None else 0.0
-        return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)])
-
-
 def _effective_harbor_agent(config: HarborRuntimeConfig | None) -> str | None:
     """The agent a run will actually use, mirroring ``run_job``'s resolution order.
 
@@ -1365,7 +1344,7 @@ def discover_harbor_tasks(dataset_path: str | Path) -> list[AgentEvalTask]:
                 # lives in `inputs["instruction"]`.
                 intent=task_name,
                 inputs={"instruction": instruction},
-                metrics=[HarborRewardMetric()],
+                metrics=[_harbor_reward_metric()],
                 metadata={"harbor_dataset_path": str(dataset_path), "harbor_task_dir": str(task_dir)},
             )
         )
@@ -1493,3 +1472,24 @@ def reward_payload_from_result(
     "run_harbor_eval",
     "scoped_harbor_agent_import",
 ]
+
+
+def _harbor_reward_metric() -> "HarborRewardMetric":
+    """Build the default reward metric, importing it lazily to keep this module light."""
+    from nemo_evaluator_sdk.metrics.runner_rewards import HarborRewardMetric
+
+    return HarborRewardMetric()
+
+
+def __getattr__(name: str) -> object:
+    """Re-export ``HarborRewardMetric`` without importing the metric stack at module scope.
+
+    Defining it here would pull ``MetricBase`` and the dataset-schema machinery (jinja2,
+    jsonschema) onto the optimizer's light import path. See
+    ``nemo_evaluator_sdk.metrics.runner_rewards``.
+    """
+    if name == "HarborRewardMetric":
+        from nemo_evaluator_sdk.metrics.runner_rewards import HarborRewardMetric
+
+        return HarborRewardMetric
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/enums.py

@@ -36,6 +36,15 @@ class MetricType(str, Enum):
     NOISE_SENSITIVITY = "noise_sensitivity"
     TUNABLE_RAG_EVALUATOR = "tunable-rag-evaluator"
 
+    # Runner-owned rewards: the runner scores, these surface it.
+    GYM_REWARD = "gym_reward"
+    HARBOR_REWARD = "harbor_reward"
+
+    # Agent-eval scoring over trial metadata and evidence.
+    AGENT_PHASE_SUCCESS = "agent_phase_success"
+    EVIDENCE_PRESENCE = "evidence_presence"
+    SKILL_USED = "skill_used"
+
     SYSTEM = "system"