Gate agentic inline accuracy on the entry the client writes

README.md

@@ -529,14 +529,20 @@ keeps the Offline requirement in force rather than letting one mislabelled point
 it off.
 
 
+Accuracy is **required** once per mandatory region, not at every point:
+`accuracy-coverage` asks for one result in each of the four mandatory bands (plus the
+Offline point for a non-agentic benchmark), and a band with several points needs only
+one of them to carry accuracy. The gates below then judge whatever results a submission
+includes, so every result present must clear.
+
 Accuracy is gated differently too. §15's gate folds every dataset of a point into one
 sample-weighted score per metric; the agentic benchmarks gate three quantities that do
 not reduce that way, so for a recognised agentic model it stands down in favour of the
 three rules above:
 
 | Quantity | Source | Aggregation |
 |---|---|---|
-| Inline accuracy | `agentic_combined` in the accuracy results | per point — every point must clear |
+| Inline accuracy | the `dataset_type: performance` entry in the accuracy results (named `performance` by the client) | each result on its own — every point that carries one must clear. Only one accuracy run per mandatory region is required (`accuracy-coverage`); points without one are not gated |
 | SWE-bench accuracy | `swe_bench` in the accuracy results | **mean-of-4** across the mandatory regions (§4.3's multi-turn branch) |
 | OSL per-turn mean | `output_sequence_lengths_full_run.output_sequence_lengths.avg` in `result_summary.json` | per point, against a range |
 

src/submission_checker/agentic_targets.py

@@ -13,7 +13,11 @@
 point into one sample-weighted score per metric and gates each point against it. The
 agentic benchmarks gate three quantities that do not reduce that way:
 
-* **Inline accuracy** — per point, from the ``agentic_combined`` performance dataset.
+* **Inline accuracy** — each result on its own, from the entry the client scores on
+  the performance run (``dataset_type: performance``; see
+  :meth:`~submission_checker.models.AccuracyResult.performance_dataset`). Every point
+  that carries one must clear; only one per mandatory region is required
+  (``accuracy-coverage``).
 * **SWE-bench accuracy** — **mean-of-N across points**, not per point: "average one
   SWE-bench accuracy result from each of the four mandatory regions (``N = 4``), then
   compare that mean with the model-specific SWE-bench threshold". This is §4.3's
@@ -33,18 +37,13 @@
 from dataclasses import dataclass
 
 __all__ = [
-    "INLINE_DATASET",
     "OSL_FULL_RUN_FIELD",
     "SWEBENCH_DATASET",
     "SWEBENCH_MEAN_OF_N",
     "AgenticTargets",
     "get_agentic_targets",
 ]
 
-#: Performance dataset carrying the inline accuracy score
-#: (``accuracy_config.eval_method: agentic_inference_inline``).
-INLINE_DATASET = "agentic_combined"
-
 #: Accuracy dataset scored by ``swe_bench_scorer``.
 SWEBENCH_DATASET = "swe_bench"
 
@@ -68,7 +67,7 @@ class AgenticTargets:
 
     #: Model name as §3.2 publishes it, for messages.
     name: str
-    #: Inline accuracy floor, per point, on a 0–100 scale.
+    #: Inline accuracy floor, for each point that reports one, on a 0–100 scale.
     inline_min: float | None
     #: SWE-bench floor for the mean of the N results, on a 0–100 scale.
     swebench_min: float | None

src/submission_checker/data/messages.yaml

@@ -114,6 +114,8 @@ accuracy-valid:
     entry-not-mapping: each accuracy_scores entry must be an object
     entry-unnamed: each accuracy_scores entry needs a non-empty dataset_name
     dataset-duplicate: dataset {name!r} appears more than once
+    performance-duplicate: 'more than one entry has dataset_type: performance ({names}); the client scores
+      the performance run once'
     score-missing: dataset {name!r} has no score
     alias-conflict:
       text: dataset {name!r} gives {native} and {canonical} different values
@@ -130,8 +132,9 @@ agentic-accuracy-inline:
   title: Agentic accuracy inline
   spec: §4.3
   messages:
-    warn: 'No point reports a `{inline_dataset}` accuracy score; official agentic submissions set `accuracy_config.eval_method:
-      agentic_inference_inline` on the performance dataset'
+    warn: 'No point reports an inline accuracy score (an accuracy entry with `dataset_type: {performance_dataset_type}`);
+      official agentic submissions set `accuracy_config.eval_method: agentic_inference_inline` on the performance
+      dataset'
     fail: 'r{concurrency}: inline accuracy {score:.2f} < {inline_min} required for {targets_name}'
     pass: 'r{concurrency}: inline accuracy {score:.2f} ≥ {inline_min}'
 agentic-accuracy-swebench:

src/submission_checker/models/aggregate/context.py

@@ -13,14 +13,13 @@
 
 from ...accuracy_targets import get_thresholds
 from ...agentic_targets import (
-    INLINE_DATASET,
     OSL_FULL_RUN_FIELD,
     SWEBENCH_DATASET,
     SWEBENCH_MEAN_OF_N,
     AgenticTargets,
     get_agentic_targets,
 )
-from ..file.accuracy import AccuracyResult
+from ..file.accuracy import PERFORMANCE_DATASET_TYPE, AccuracyResult
 from ..file.point_config import (
     LOAD_PATTERN_AGENTIC,
     OFFLINE_DEDICATED,
@@ -657,11 +656,22 @@ def _check_agentic_accuracy(self) -> ModelContext:
         return self
 
     def _gate_agentic_inline(self, targets: AgenticTargets) -> None:
-        """Inline accuracy, per point: "Every … submitted Pareto point must satisfy"."""
+        """Inline accuracy, at each point that reports it.
+
+        "Every … submitted Pareto point must satisfy" applies to the results submitted.
+        Points without one are skipped; that one result per mandatory region exists is
+        ``accuracy-coverage``'s check.
+
+        Read from the entry scored on the performance run, which the client names
+        ``performance`` rather than after the performance dataset.
+        """
         assert targets.inline_min is not None
         seen = False
         for concurrency in sorted(self.accuracy_by_point):
-            score = self._dataset_score(concurrency, INLINE_DATASET, "agentic-accuracy-inline")
+            dataset = self.accuracy_by_point[concurrency].performance_dataset()
+            if dataset is None:
+                continue
+            score = self._dataset_score(concurrency, dataset, "agentic-accuracy-inline")
             if score is None:
                 continue
             seen = True
@@ -694,7 +704,7 @@ def _gate_agentic_inline(self, targets: AgenticTargets) -> None:
                     "agentic-accuracy-inline",
                     "warn",
                     self.points_dir,
-                    inline_dataset=INLINE_DATASET,
+                    performance_dataset_type=PERFORMANCE_DATASET_TYPE,
                 )
             )
 

src/submission_checker/models/file/accuracy.py

@@ -10,10 +10,15 @@
 from ...messages import Invalid
 from ..results import CheckResult, err
 
-__all__ = ["AccuracyResult"]
+__all__ = ["PERFORMANCE_DATASET_TYPE", "AccuracyResult"]
 
 _log = logging.getLogger(__name__)
 
+#: ``dataset_type`` of the entry the client scores inline on the performance run
+#: (``accuracy_config`` on the performance dataset). The client also names that entry
+#: ``performance``, whatever the performance dataset is called in ``config.yaml``.
+PERFORMANCE_DATASET_TYPE = "performance"
+
 
 class AccuracyResult(RootModel[dict[str, dict[str, Any]]]):
     """Parsed ``accuracy/accuracy_result.json``.
@@ -79,6 +84,33 @@ def _check_not_empty(self) -> AccuracyResult:
             self._check_results.append(err("accuracy-valid", "fail", None))
         return self
 
+    @model_validator(mode="after")
+    def _check_one_performance_entry(self) -> AccuracyResult:
+        """The client scores the performance run once, so it writes one such entry."""
+        typed = [
+            name
+            for name, entry in self.root.items()
+            if entry.get("dataset_type") == PERFORMANCE_DATASET_TYPE
+        ]
+        if len(typed) > 1:
+            raise Invalid("accuracy-valid", "performance-duplicate", names=", ".join(typed))
+        return self
+
+    def performance_dataset(self) -> str | None:
+        """Name of the entry scored inline on the performance run, or ``None``.
+
+        Found by ``dataset_type``, as the client asks of consumers: an accuracy dataset
+        may itself be named ``performance``. Clients that predate ``dataset_type``
+        wrote the name alone, so an untyped ``performance`` entry still counts.
+        """
+        for name, entry in self.root.items():
+            if entry.get("dataset_type") == PERFORMANCE_DATASET_TYPE:
+                return name
+        legacy = self.root.get(PERFORMANCE_DATASET_TYPE)
+        if legacy is not None and "dataset_type" not in legacy:
+            return PERFORMANCE_DATASET_TYPE
+        return None
+
     #: Per-entry bookkeeping keys that are not accuracy metrics.
     _META_KEYS = frozenset(
         {

tests/submission_checker/test_checks_agentic_accuracy.py

@@ -38,7 +38,11 @@ def _accuracy(
     scale = 1.0 if raw else 0.01
     root: dict[str, dict[str, object]] = {}
     if inline is not None:
-        root["agentic_combined"] = {"score": inline * scale, "num_samples": 613}
+        root["performance"] = {
+            "score": inline * scale,
+            "num_samples": 613,
+            "dataset_type": "performance",
+        }
     if swebench is not None:
         root["swe_bench"] = {"score": swebench * scale, "num_samples": 200}
     return AccuracyResult(root)
@@ -182,6 +186,48 @@ def test_absent_warns(self, tmp_path: Path) -> None:
         assert hits and hits[0].severity == Severity.WARNING
 
 
+@pytest.mark.unit
+class TestInlineEntry:
+    """Which accuracy entry carries the inline score."""
+
+    def test_native_client_report_is_gated(self, tmp_path: Path) -> None:
+        """The client's own shape: a list entry named and typed ``performance``."""
+        report = AccuracyResult.model_validate(
+            {
+                "accuracy_scores": [
+                    {"dataset_name": "swe_bench", "score": 0.715, "dataset_type": "accuracy"},
+                    {"dataset_name": "performance", "score": 0.50, "dataset_type": "performance"},
+                ]
+            }
+        )
+        ctx = _ctx(tmp_path, model_name="qwen3_6-35b-a3b", accuracy_by_point={16: report})
+        errors = _errors(ctx, "agentic-accuracy-inline")
+        assert errors and "50.00 <" in errors[0].message
+
+    def test_found_by_type_not_name(self, tmp_path: Path) -> None:
+        report = AccuracyResult({"agentic_coding": {"score": 0.50, "dataset_type": "performance"}})
+        ctx = _ctx(tmp_path, accuracy_by_point={16: report})
+        assert _errors(ctx, "agentic-accuracy-inline")
+
+    def test_untyped_performance_entry_is_read(self, tmp_path: Path) -> None:
+        """Clients that predate ``dataset_type`` wrote the name alone."""
+        report = AccuracyResult({"performance": {"score": 0.50}})
+        ctx = _ctx(tmp_path, accuracy_by_point={16: report})
+        assert _errors(ctx, "agentic-accuracy-inline")
+
+    def test_accuracy_dataset_named_performance_is_not_inline(self, tmp_path: Path) -> None:
+        report = AccuracyResult({"performance": {"score": 0.50, "dataset_type": "accuracy"}})
+        ctx = _ctx(tmp_path, accuracy_by_point={16: report})
+        hits = _hits(ctx, "agentic-accuracy-inline")
+        assert [h.severity for h in hits] == [Severity.WARNING]
+
+    def test_untyped_entry_under_another_name_is_not_inline(self, tmp_path: Path) -> None:
+        report = AccuracyResult({"agentic_combined": {"score": 0.50}})
+        ctx = _ctx(tmp_path, accuracy_by_point={16: report})
+        hits = _hits(ctx, "agentic-accuracy-inline")
+        assert [h.severity for h in hits] == [Severity.WARNING]
+
+
 @pytest.mark.unit
 class TestSweBenchMeanOfN:
     """§4.3 multi-turn: the mean must clear; "individual results need not"."""

tests/submission_checker/test_native_formats.py

@@ -80,6 +80,10 @@ def test_native_accuracy_reads_each_dataset_and_preserves_file(tmp_path, embedde
         [{"dataset_name": "  ", "score": 1}],
         [{"dataset_name": "missing-score", "total_samples": 200}],
         [{"dataset_name": "conflict", "score": 1, "unit_samples": 200, "num_samples": 100}],
+        [
+            {"dataset_name": "performance", "score": 1, "dataset_type": "performance"},
+            {"dataset_name": "perf-2", "score": 0, "dataset_type": "performance"},
+        ],
         ["not-an-entry"],
         "not-a-list-or-mapping",
     ],