feat(evaluator): let evaluations pin a taskset revision

docs/evaluator/manage-tasks-tasksets.mdx

@@ -290,7 +290,10 @@ input_spec = AgentEvalInputSpec(
 
 When the job runs, the taskset reference is resolved like this:
 
-- Each member is loaded at the **revision pinned in the taskset**, not the task's current tip.
+- The **taskset revision** the ref names is loaded — the current one unless the ref pins a revision
+  (see [Pin the taskset itself](#pin-the-taskset-itself)).
+- Each member of that revision is loaded at the **revision pinned in the taskset**, not the task's
+  current tip.
 - Metric references on those members are hydrated into runnable metrics, the same as for inline tasks.
 - Re-running the same taskset therefore evaluates the same content, even if a member has been
   republished since.
@@ -320,11 +323,39 @@ results flow.
 The inline form remains available for one-off tasks — swap `tasks=TasksetRef(...)` for
 `tasks=[AgentEvalTaskInput(...), ...]`.
 
-<Note>
-A `TasksetRef` names the taskset's current revision; it cannot yet carry a `#<tag-or-digest>`
-fragment of its own. Member content is pinned, so a re-run always grades the same task content — but
-if the taskset itself is replaced, a re-submitted spec expands the new membership.
-</Note>
+### Pin the taskset itself
+
+A bare `TasksetRef` expands the taskset's current revision, so it follows the suite forward as
+members are added or removed. Add a `#<tag-or-digest>` fragment to pin the grouping too:
+
+```python
+# The digest lives on the revision, not on the taskset record — revisions come back newest first.
+current = tasksets.list_revisions("geography-suite").data[0]
+
+# Follows the suite forward — new members are picked up on the next run.
+tracking = TasksetRef("default/geography-suite")
+
+# Frozen: this exact membership, regardless of later `replace` calls.
+pinned = TasksetRef(f"default/geography-suite#{current.content_hash}")
+
+# A tag works the same way, and can be moved deliberately when you bless a new suite.
+blessed = TasksetRef("default/geography-suite#blessed")
+```
+
+The two pins compose, and they cover different things:
+
+| | Member content | Which members |
+|---|---|---|
+| `TasksetRef("suite")` | pinned | current revision |
+| `TasksetRef("suite#<digest>")` | pinned | pinned |
+
+Member content is digest-pinned inside every revision, so even a bare ref grades the same task
+content across re-runs. Pinning the taskset is what additionally holds *membership* still across a
+`replace` that adds or drops a task — which is what you want when a benchmark number has to stay
+comparable.
+
+A fragment that no longer resolves fails the evaluation rather than falling back to the current
+revision.
 
 <Note>
 Stored tasks carry no grader-only `reference` (held-out ground truth): that field lives only on inline
@@ -401,6 +432,7 @@ The SDK resources are a thin client over the Evaluator plugin REST API, mounted
 | Missing or duplicate task reference (taskset) | `422` | Members must exist, and must resolve to distinct tasks. |
 | Reserved or malformed tag name | `422` | `latest` cannot be moved by hand; a digest-shaped tag is refused. |
 | Retrieving or deleting an unknown name or revision | `404` | Applies to both records and revisions. |
+| `TasksetRef` pinning a revision that no longer resolves | job fails | Expansion refuses rather than falling back to the current revision. |
 | `DELETE` on a task a taskset pins | `204` | Not prevented; the taskset's reference dangles and fails on read. |
 
 ## Related Topics

plugins/nemo-evaluator/src/nemo_evaluator/api/schemas.py

@@ -151,9 +151,9 @@ class MetricInline(BaseModel):
 # (``#latest``, ``#candidate``) or a full 64-char content digest.
 #
 # Deliberately a sibling of ``_ENTITY_REF_PATTERN`` rather than a widening of it: that constant is
-# shared by ``MetricRef`` and ``TasksetRef``, neither of which has revisions yet, and admitting a
-# fragment there would accept input nothing is built to resolve. ``TasksetRef`` moves onto this
-# pattern when taskset revisions are addressable; ``MetricRef`` when (if) metrics gain revisions.
+# still shared by ``MetricRef``, which has no revisions, and admitting a fragment there would accept
+# input nothing is built to resolve. ``TaskRef`` and ``TasksetRef`` both use this pattern, since both
+# name revisioned records; ``MetricRef`` joins them when (if) metrics gain revisions.
 _SUBENTITY_REF_PATTERN = rf"^[\w\-.]+(/[\w\-.]+)?(#{REF_FRAGMENT_CHARSET})?$"
 
 #: The fragment separator for sub-entity references. Matches the fileset/job ref convention.
@@ -229,15 +229,21 @@ class TaskRef(RootModel[str]):
 
 
 class TasksetRef(RootModel[str]):
-    """Reference to a persisted taskset (format: ``workspace/name`` or ``name``).
+    """Reference to a persisted taskset (format: ``workspace/name`` or ``name``, optionally ``#rev``).
 
     Same shape and charset as :class:`TaskRef`. Lets an evaluation reference a stored taskset in place
     of an inline task list; the taskset's member tasks are loaded and expanded during spec resolution.
+
+    An optional ``#`` fragment pins the taskset revision to expand — a tag or a full content digest,
+    with an absent fragment meaning ``latest``. Membership is digest-pinned within a revision, so a
+    bare ref already grades identical task *content* across re-runs; pinning the taskset as well is
+    what fixes the *membership* too, across a ``replace`` that adds or drops a member.
     """
 
     root: str = Field(
-        pattern=_ENTITY_REF_PATTERN,
-        description="Reference to a stored taskset (format: workspace/taskset-name, or taskset-name in the job workspace).",
+        pattern=_SUBENTITY_REF_PATTERN,
+        description="Reference to a stored taskset (format: workspace/taskset-name, or taskset-name in the "
+        "job workspace), optionally pinned to a revision with '#<tag-or-digest>'.",
     )
 
 

plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py

@@ -18,8 +18,8 @@
 
 from typing import cast
 
-from nemo_evaluator.api.schemas import TasksetRef, parse_entity_ref, parse_subentity_ref
-from nemo_evaluator.entities import TaskEntity, TaskRevisionEntity, TasksetEntity
+from nemo_evaluator.api.schemas import TasksetRef, parse_subentity_ref
+from nemo_evaluator.entities import TaskEntity, TaskRevisionEntity, TasksetEntity, TasksetRevisionEntity
 from nemo_evaluator.jobs.agent_spec import AgentEvalTaskInput
 from nemo_evaluator.revisions import RevisionNotFoundError, get_revision
 from nemo_platform_plugin.entities import EntityClientProtocol
@@ -49,10 +49,11 @@ def _entity_to_task_input(entity: TaskEntity, revision: TaskRevisionEntity) -> A
     )
 
 
-#: Expanding a taskset reads three entity types through one client — the taskset head, each member
-#: task's head, and the pinned revision of each member. Python has no intersection types, so the
-#: parameter is annotated at one of them and the other two are taken as typed views of the same
-#: object; the concrete client's methods are generic over the entity type and satisfy all three.
+#: Expanding a taskset reads four entity types through one client — the taskset head, its pinned
+#: revision, each member task's head, and the pinned revision of each member. Python has no
+#: intersection types, so the parameter is annotated at one of them and the rest are taken as typed
+#: views of the same object; the concrete client's methods are generic over the entity type and
+#: satisfy all four.
 TasksetStoreProtocol = EntityClientProtocol[TasksetEntity]
 
 
@@ -66,6 +67,12 @@ async def resolve_taskset_ref(
 
     Loading needs only the entity store (metrics stay as refs, resolved downstream), so unlike
     metric-ref resolution this does not require an async SDK / file I/O.
+
+    The ref may pin a taskset revision (``suite#<tag-or-digest>``); an absent fragment means
+    ``latest``. Both paths go through :func:`get_revision` rather than reading the head's own
+    ``tasks``, because a head and its ``latest`` revision are guaranteed to agree and resolving one
+    way for pinned refs and another way for bare ones would make the two drift apart on the next
+    bug. It also buys content verification for the bare case for free.
     """
     if entity_client is None:
         raise ValueError(
@@ -74,8 +81,9 @@ async def resolve_taskset_ref(
         )
     task_store = cast(EntityClientProtocol[TaskEntity], entity_client)
     revision_store = cast(EntityClientProtocol[TaskRevisionEntity], entity_client)
+    taskset_revision_store = cast(EntityClientProtocol[TasksetRevisionEntity], entity_client)
 
-    ref_workspace, name = parse_entity_ref(ref.root, workspace)
+    ref_workspace, name, taskset_fragment = parse_subentity_ref(ref.root, workspace)
     try:
         taskset = await entity_client.get(TasksetEntity, name=name, workspace=ref_workspace)
     except NemoEntityNotFoundError as exc:
@@ -85,12 +93,20 @@ async def resolve_taskset_ref(
             "or pass an inline task list instead."
         ) from exc
 
-    if not taskset.tasks:
+    # Expand the *pinned* taskset revision. Membership is digest-pinned within a revision, so a bare
+    # ref already grades identical task content — but a ``replace`` that adds or drops a member
+    # changes the head, and only pinning the taskset itself holds membership steady across that.
+    try:
+        taskset_revision = await get_revision(taskset_revision_store, TasksetRevisionEntity, taskset, taskset_fragment)
+    except RevisionNotFoundError as exc:
+        raise ValueError(f"Taskset reference '{ref.root}' names a revision that does not resolve: {exc}") from exc
+
+    if not taskset_revision.tasks:
         raise ValueError(f"Taskset '{ref.root}' has no member tasks; an agent evaluation needs at least one task.")
 
     tasks: list[AgentEvalTaskInput] = []
     seen_ids: set[str] = set()
-    for task_ref in taskset.tasks:
+    for task_ref in taskset_revision.tasks:
         task_workspace, task_name, fragment = parse_subentity_ref(task_ref.root, ref_workspace)
         try:
             entity = await task_store.get(TaskEntity, name=task_name, workspace=task_workspace)

plugins/nemo-evaluator/tests/test_subentity_refs.py

@@ -90,10 +90,21 @@ def test_task_ref_rejects_malformed_fragments(ref: str) -> None:
         TaskRef(ref)
 
 
-@pytest.mark.parametrize("ref_type", [MetricRef, TasksetRef])
-def test_sibling_ref_types_still_reject_fragments(ref_type: type) -> None:
-    """The fragment pattern is a sibling, not a widening of the shared constant: metrics and
-    tasksets have no revisions yet, so admitting a fragment would accept input nothing resolves.
-    They move onto it when they gain revisions — deliberately, at that point."""
+def test_metric_ref_still_rejects_fragments() -> None:
+    """The fragment pattern is a sibling, not a widening of the shared constant: metrics have no
+    revisions, so admitting a fragment would accept input nothing resolves. ``MetricRef`` moves onto
+    it when metrics gain revisions — deliberately, at that point."""
     with pytest.raises(ValidationError):
-        ref_type(f"other/thing#{_DIGEST}")
+        MetricRef(f"other/thing#{_DIGEST}")
+
+
+@pytest.mark.parametrize("ref", ["suite", "other/suite", "suite#latest", "suite#blessed", f"other/suite#{_DIGEST}"])
+def test_taskset_ref_accepts_fragments(ref: str) -> None:
+    """Tasksets are revisioned, so a ref may pin the revision to expand — same shape as ``TaskRef``."""
+    assert TasksetRef(ref).root == ref
+
+
+@pytest.mark.parametrize("ref", ["suite#one#two", "suite#bad/frag", "#latest", "suite name#latest"])
+def test_taskset_ref_rejects_malformed_fragments(ref: str) -> None:
+    with pytest.raises(ValidationError):
+        TasksetRef(ref)