docs(evaluator): update the evaluator skill for the task spec shape

Split out of #1071 so that PR is not held behind the NVSkills gate. The gate
cannot currently pass for this repo: tier 3 is invoked with `--env-mode local`,
whose bubblewrap sandbox fails its smoke test on the nvcarps runners, so nothing
is evaluated and the gate blocks on empty coverage. This PR carries the whole
cost of that, and can wait for the infrastructure fix without blocking the
storage change.

`store_resources` moves to the discriminated `spec`, and `resources.md` teaches
held-out ground truth on a *stored* task rather than an inline one — the skill
steered users to `AgentEvalTaskInput` only because the stored schema had no
`reference` field, and #1071 gives it one. Routing them back to inline would
cost them tasksets and revision pinning for no reason.

Until this lands, the published skill documents the pre-`spec` shape, which no
longer validates against #1071. Merge promptly once the gate is healthy.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Signed-off-by: Sandy Chapman <schapman@nvidia.com>

Author: SandyChapman

plugins/nemo-evaluator/tests/test_skill_examples.py

@@ -445,13 +445,20 @@ def test_multiple_metric_platform_submission_uses_cli() -> None:
     assert "nemo evaluator evaluate submit --spec-file multi-metric.json" in section
 
 
-def test_resources_show_inline_task_before_held_out_reference_guidance() -> None:
+def test_resources_show_a_stored_task_carrying_held_out_reference() -> None:
+    """Held-out ground truth belongs on a *stored* task, so it survives taskset expansion.
+
+    The skill used to steer users to an inline ``AgentEvalTaskInput`` because the stored spec had no
+    ``reference`` field. It has one now, and routing them back to inline would cost them tasksets
+    and revision pinning for no reason.
+    """
     reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/resources.md").read_text(encoding="utf-8")
 
-    example_position = reference.index("inline_task = AgentEvalTaskInput(")
+    example_position = reference.index('"capital-france-graded"')
     guidance_position = reference.index("Stored tasks keep metric references.")
     assert example_position < guidance_position
     assert 'reference={"expected": "Paris"}' in reference
+    assert "EvaluatorTaskDefinition(" in reference
 
 
 def test_agent_evaluation_shows_how_to_retrieve_stored_trials() -> None:

skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py

@@ -58,6 +58,7 @@ def submit_and_collect(client: Any, output_dir: Path) -> tuple[Any, Path]:
 def store_resources(client: Any) -> None:
     """Store one metric, task, and taskset."""
     from nemo_evaluator.api.schemas import (
+        EvaluatorTaskDefinition,
         MetricRef,
         TaskInput,
         TaskInputs,
@@ -72,14 +73,17 @@ def store_resources(client: Any) -> None:
     client.evaluator.tasks.create(
         "capital-france",
         task=TaskInput(
-            intent="Name the capital of France.",
-            inputs=TaskInputs(instruction="What is the capital of France?"),
-            metrics=[MetricRef("default/answer-exact")],
+            spec=EvaluatorTaskDefinition(
+                kind="evaluator",
+                intent="Name the capital of France.",
+                inputs=TaskInputs(instruction="What is the capital of France?"),
+                metrics=[MetricRef("answer-exact")],
+            ),
         ),
     )
     client.evaluator.tasksets.create(
         "geography",
-        taskset=TasksetInput(tasks=[TaskRef("default/capital-france")]),
+        taskset=TasksetInput(tasks=[TaskRef("capital-france")]),
     )
 
 

skills/nemo-evaluator-plugin/references/resources.md

@@ -20,6 +20,7 @@ new versioned name.
 
 ```python
 from nemo_evaluator.api.schemas import (
+    EvaluatorTaskDefinition,
     MetricRef,
     TaskInput,
     TaskInputs,
@@ -43,27 +44,29 @@ client.evaluator.metrics.create(
 client.evaluator.tasks.create(
     "capital-france",
     task=TaskInput(
-        intent="Name the capital of France.",
-        inputs=TaskInputs(instruction="What is the capital of France?"),
-        metrics=[MetricRef("default/answer-exact")],
+        spec=EvaluatorTaskDefinition(
+            kind="evaluator",
+            intent="Name the capital of France.",
+            inputs=TaskInputs(instruction="What is the capital of France?"),
+            metrics=[MetricRef("answer-exact")],
+        ),
     ),
 )
 
 client.evaluator.tasksets.create(
     "geography",
     taskset=TasksetInput(
         description="Geography smoke tasks.",
-        tasks=[TaskRef("default/capital-france")],
+        tasks=[TaskRef("capital-france")],
     ),
 )
 ```
 
-For a task that needs held-out ground truth invisible to the agent, keep the reference on an
-inline `AgentEvalTaskInput` and use a metric that reads it:
+For a task that needs held-out ground truth invisible to the agent, put it in `reference` and
+use a metric that reads it. This works on a stored task, so it survives into taskset-driven runs:
 
 ```python
-from nemo_evaluator.api.schemas import MetricRef, TaskInputs
-from nemo_evaluator.jobs.agent_spec import AgentEvalTaskInput
+from nemo_evaluator.api.schemas import EvaluatorTaskDefinition, MetricRef, TaskInput, TaskInputs
 from nemo_evaluator_sdk import ExactMatchMetric
 
 client.evaluator.metrics.create(
@@ -74,19 +77,28 @@ client.evaluator.metrics.create(
     ),
 )
 
-inline_task = AgentEvalTaskInput(
-    id="capital-france",
-    intent="Name the capital of France.",
-    inputs=TaskInputs(instruction="What is the capital of France?"),
-    reference={"expected": "Paris"},
-    metrics=[MetricRef("default/answer-from-reference")],
+client.evaluator.tasks.create(
+    "capital-france-graded",
+    task=TaskInput(
+        spec=EvaluatorTaskDefinition(
+            kind="evaluator",
+            intent="Name the capital of France.",
+            inputs=TaskInputs(instruction="What is the capital of France?"),
+            reference={"expected": "Paris"},
+            metrics=[MetricRef("answer-from-reference")],
+        ),
+    ),
 )
 ```
 
+`reference` is surfaced to metrics but never seeded into the agent's workspace or shown to the
+agent, so a metric can grade against artifacts the agent cannot edit. It is held out from the
+*agent*, not from the API — anyone who can read the task can read it. It is covered by the revision
+digest, so changing ground truth publishes a new revision.
+
 Stored tasks keep metric references. Inline task metrics are normalized into
-content-addressed derived metrics. The stored-task example uses an output-only
-metric because stored tasks do not carry the grader-only `reference` field; use
-an inline `AgentEvalTaskInput` when held-out per-task data is required.
+content-addressed derived metrics. The same `reference` field is available on an inline
+`AgentEvalTaskInput` for one-off submissions.
 
 ## Retrieve, list, and delete