From b889565c6f8fcdabb49785f7c095d860a28ca7b3 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 8 Oct 2026 14:44:34 +0530 Subject: [PATCH 1/2] Gate agentic inline accuracy on the entry the client writes The inline gate looked up an accuracy entry named agentic_combined, which is the performance dataset's name in the Kimi K3 and DSV4 example configs. The client never writes that name: it records the inline score under dataset_name "performance", tagged dataset_type "performance", whatever the performance dataset is called. No real submission ever reached the gate, so inline accuracy was never checked for Kimi K3, Qwen3.6-35B-A3B or DeepSeek-V4.1-Flash, and a failing score only raised a warning. Find the entry by dataset_type, as the client asks of consumers. An untyped entry named performance, from clients that predate dataset_type, still counts. An accuracy file with more than one dataset_type performance entry is now invalid; the client scores the performance run once and refuses more than one performance dataset. Tests use the client's own output shape. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- src/submission_checker/agentic_targets.py | 9 ++-- src/submission_checker/data/messages.yaml | 7 ++- .../models/aggregate/context.py | 16 +++++-- .../models/file/accuracy.py | 34 ++++++++++++- .../test_checks_agentic_accuracy.py | 48 ++++++++++++++++++- .../submission_checker/test_native_formats.py | 4 ++ 7 files changed, 104 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b1e873a..494821b 100644 --- a/README.md +++ b/README.md @@ -536,7 +536,7 @@ three rules above: | Quantity | Source | Aggregation | |---|---|---| -| Inline accuracy | `agentic_combined` in the accuracy results | per point — every point must clear | +| Inline accuracy | the `dataset_type: performance` entry in the accuracy results (named `performance` by the client) | per point — every point must clear | | SWE-bench accuracy | `swe_bench` in the accuracy results | **mean-of-4** across the mandatory regions (§4.3's multi-turn branch) | | OSL per-turn mean | `output_sequence_lengths_full_run.output_sequence_lengths.avg` in `result_summary.json` | per point, against a range | diff --git a/src/submission_checker/agentic_targets.py b/src/submission_checker/agentic_targets.py index beb8b3f..c4ffc75 100644 --- a/src/submission_checker/agentic_targets.py +++ b/src/submission_checker/agentic_targets.py @@ -13,7 +13,9 @@ point into one sample-weighted score per metric and gates each point against it. The agentic benchmarks gate three quantities that do not reduce that way: -* **Inline accuracy** — per point, from the ``agentic_combined`` performance dataset. +* **Inline accuracy** — per point, from the entry the client scores on the performance + run (``dataset_type: performance``; see + :meth:`~submission_checker.models.AccuracyResult.performance_dataset`). * **SWE-bench accuracy** — **mean-of-N across points**, not per point: "average one SWE-bench accuracy result from each of the four mandatory regions (``N = 4``), then compare that mean with the model-specific SWE-bench threshold". This is §4.3's @@ -33,7 +35,6 @@ from dataclasses import dataclass __all__ = [ - "INLINE_DATASET", "OSL_FULL_RUN_FIELD", "SWEBENCH_DATASET", "SWEBENCH_MEAN_OF_N", @@ -41,10 +42,6 @@ "get_agentic_targets", ] -#: Performance dataset carrying the inline accuracy score -#: (``accuracy_config.eval_method: agentic_inference_inline``). -INLINE_DATASET = "agentic_combined" - #: Accuracy dataset scored by ``swe_bench_scorer``. SWEBENCH_DATASET = "swe_bench" diff --git a/src/submission_checker/data/messages.yaml b/src/submission_checker/data/messages.yaml index 21233a7..c560a80 100644 --- a/src/submission_checker/data/messages.yaml +++ b/src/submission_checker/data/messages.yaml @@ -114,6 +114,8 @@ accuracy-valid: entry-not-mapping: each accuracy_scores entry must be an object entry-unnamed: each accuracy_scores entry needs a non-empty dataset_name dataset-duplicate: dataset {name!r} appears more than once + performance-duplicate: 'more than one entry has dataset_type: performance ({names}); the client scores + the performance run once' score-missing: dataset {name!r} has no score alias-conflict: text: dataset {name!r} gives {native} and {canonical} different values @@ -130,8 +132,9 @@ agentic-accuracy-inline: title: Agentic accuracy inline spec: §4.3 messages: - warn: 'No point reports a `{inline_dataset}` accuracy score; official agentic submissions set `accuracy_config.eval_method: - agentic_inference_inline` on the performance dataset' + warn: 'No point reports an inline accuracy score (an accuracy entry with `dataset_type: {performance_dataset_type}`); + official agentic submissions set `accuracy_config.eval_method: agentic_inference_inline` on the performance + dataset' fail: 'r{concurrency}: inline accuracy {score:.2f} < {inline_min} required for {targets_name}' pass: 'r{concurrency}: inline accuracy {score:.2f} ≥ {inline_min}' agentic-accuracy-swebench: diff --git a/src/submission_checker/models/aggregate/context.py b/src/submission_checker/models/aggregate/context.py index 07f8b51..3c3a16f 100644 --- a/src/submission_checker/models/aggregate/context.py +++ b/src/submission_checker/models/aggregate/context.py @@ -13,14 +13,13 @@ from ...accuracy_targets import get_thresholds from ...agentic_targets import ( - INLINE_DATASET, OSL_FULL_RUN_FIELD, SWEBENCH_DATASET, SWEBENCH_MEAN_OF_N, AgenticTargets, get_agentic_targets, ) -from ..file.accuracy import AccuracyResult +from ..file.accuracy import PERFORMANCE_DATASET_TYPE, AccuracyResult from ..file.point_config import ( LOAD_PATTERN_AGENTIC, OFFLINE_DEDICATED, @@ -657,11 +656,18 @@ def _check_agentic_accuracy(self) -> ModelContext: return self def _gate_agentic_inline(self, targets: AgenticTargets) -> None: - """Inline accuracy, per point: "Every … submitted Pareto point must satisfy".""" + """Inline accuracy, per point: "Every … submitted Pareto point must satisfy". + + Read from the entry scored on the performance run, which the client names + ``performance`` rather than after the performance dataset. + """ assert targets.inline_min is not None seen = False for concurrency in sorted(self.accuracy_by_point): - score = self._dataset_score(concurrency, INLINE_DATASET, "agentic-accuracy-inline") + dataset = self.accuracy_by_point[concurrency].performance_dataset() + if dataset is None: + continue + score = self._dataset_score(concurrency, dataset, "agentic-accuracy-inline") if score is None: continue seen = True @@ -694,7 +700,7 @@ def _gate_agentic_inline(self, targets: AgenticTargets) -> None: "agentic-accuracy-inline", "warn", self.points_dir, - inline_dataset=INLINE_DATASET, + performance_dataset_type=PERFORMANCE_DATASET_TYPE, ) ) diff --git a/src/submission_checker/models/file/accuracy.py b/src/submission_checker/models/file/accuracy.py index b00dd8e..88a1be0 100644 --- a/src/submission_checker/models/file/accuracy.py +++ b/src/submission_checker/models/file/accuracy.py @@ -10,10 +10,15 @@ from ...messages import Invalid from ..results import CheckResult, err -__all__ = ["AccuracyResult"] +__all__ = ["PERFORMANCE_DATASET_TYPE", "AccuracyResult"] _log = logging.getLogger(__name__) +#: ``dataset_type`` of the entry the client scores inline on the performance run +#: (``accuracy_config`` on the performance dataset). The client also names that entry +#: ``performance``, whatever the performance dataset is called in ``config.yaml``. +PERFORMANCE_DATASET_TYPE = "performance" + class AccuracyResult(RootModel[dict[str, dict[str, Any]]]): """Parsed ``accuracy/accuracy_result.json``. @@ -79,6 +84,33 @@ def _check_not_empty(self) -> AccuracyResult: self._check_results.append(err("accuracy-valid", "fail", None)) return self + @model_validator(mode="after") + def _check_one_performance_entry(self) -> AccuracyResult: + """The client scores the performance run once, so it writes one such entry.""" + typed = [ + name + for name, entry in self.root.items() + if entry.get("dataset_type") == PERFORMANCE_DATASET_TYPE + ] + if len(typed) > 1: + raise Invalid("accuracy-valid", "performance-duplicate", names=", ".join(typed)) + return self + + def performance_dataset(self) -> str | None: + """Name of the entry scored inline on the performance run, or ``None``. + + Found by ``dataset_type``, as the client asks of consumers: an accuracy dataset + may itself be named ``performance``. Clients that predate ``dataset_type`` + wrote the name alone, so an untyped ``performance`` entry still counts. + """ + for name, entry in self.root.items(): + if entry.get("dataset_type") == PERFORMANCE_DATASET_TYPE: + return name + legacy = self.root.get(PERFORMANCE_DATASET_TYPE) + if legacy is not None and "dataset_type" not in legacy: + return PERFORMANCE_DATASET_TYPE + return None + #: Per-entry bookkeeping keys that are not accuracy metrics. _META_KEYS = frozenset( { diff --git a/tests/submission_checker/test_checks_agentic_accuracy.py b/tests/submission_checker/test_checks_agentic_accuracy.py index 4e85cbb..a2c2249 100644 --- a/tests/submission_checker/test_checks_agentic_accuracy.py +++ b/tests/submission_checker/test_checks_agentic_accuracy.py @@ -38,7 +38,11 @@ def _accuracy( scale = 1.0 if raw else 0.01 root: dict[str, dict[str, object]] = {} if inline is not None: - root["agentic_combined"] = {"score": inline * scale, "num_samples": 613} + root["performance"] = { + "score": inline * scale, + "num_samples": 613, + "dataset_type": "performance", + } if swebench is not None: root["swe_bench"] = {"score": swebench * scale, "num_samples": 200} return AccuracyResult(root) @@ -182,6 +186,48 @@ def test_absent_warns(self, tmp_path: Path) -> None: assert hits and hits[0].severity == Severity.WARNING +@pytest.mark.unit +class TestInlineEntry: + """Which accuracy entry carries the inline score.""" + + def test_native_client_report_is_gated(self, tmp_path: Path) -> None: + """The client's own shape: a list entry named and typed ``performance``.""" + report = AccuracyResult.model_validate( + { + "accuracy_scores": [ + {"dataset_name": "swe_bench", "score": 0.715, "dataset_type": "accuracy"}, + {"dataset_name": "performance", "score": 0.50, "dataset_type": "performance"}, + ] + } + ) + ctx = _ctx(tmp_path, model_name="qwen3_6-35b-a3b", accuracy_by_point={16: report}) + errors = _errors(ctx, "agentic-accuracy-inline") + assert errors and "50.00 <" in errors[0].message + + def test_found_by_type_not_name(self, tmp_path: Path) -> None: + report = AccuracyResult({"agentic_coding": {"score": 0.50, "dataset_type": "performance"}}) + ctx = _ctx(tmp_path, accuracy_by_point={16: report}) + assert _errors(ctx, "agentic-accuracy-inline") + + def test_untyped_performance_entry_is_read(self, tmp_path: Path) -> None: + """Clients that predate ``dataset_type`` wrote the name alone.""" + report = AccuracyResult({"performance": {"score": 0.50}}) + ctx = _ctx(tmp_path, accuracy_by_point={16: report}) + assert _errors(ctx, "agentic-accuracy-inline") + + def test_accuracy_dataset_named_performance_is_not_inline(self, tmp_path: Path) -> None: + report = AccuracyResult({"performance": {"score": 0.50, "dataset_type": "accuracy"}}) + ctx = _ctx(tmp_path, accuracy_by_point={16: report}) + hits = _hits(ctx, "agentic-accuracy-inline") + assert [h.severity for h in hits] == [Severity.WARNING] + + def test_untyped_entry_under_another_name_is_not_inline(self, tmp_path: Path) -> None: + report = AccuracyResult({"agentic_combined": {"score": 0.50}}) + ctx = _ctx(tmp_path, accuracy_by_point={16: report}) + hits = _hits(ctx, "agentic-accuracy-inline") + assert [h.severity for h in hits] == [Severity.WARNING] + + @pytest.mark.unit class TestSweBenchMeanOfN: """§4.3 multi-turn: the mean must clear; "individual results need not".""" diff --git a/tests/submission_checker/test_native_formats.py b/tests/submission_checker/test_native_formats.py index bf39aa3..a2d1f03 100644 --- a/tests/submission_checker/test_native_formats.py +++ b/tests/submission_checker/test_native_formats.py @@ -80,6 +80,10 @@ def test_native_accuracy_reads_each_dataset_and_preserves_file(tmp_path, embedde [{"dataset_name": " ", "score": 1}], [{"dataset_name": "missing-score", "total_samples": 200}], [{"dataset_name": "conflict", "score": 1, "unit_samples": 200, "num_samples": 100}], + [ + {"dataset_name": "performance", "score": 1, "dataset_type": "performance"}, + {"dataset_name": "perf-2", "score": 0, "dataset_type": "performance"}, + ], ["not-an-entry"], "not-a-list-or-mapping", ], From d7dc814b95c99cc3c1bdfe07e7babdb4b312a7ee Mon Sep 17 00:00:00 2001 From: Arav Agarwal Date: Thu, 8 Oct 2026 13:20:51 -0400 Subject: [PATCH 2/2] Say accuracy is required once per region, not at every point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README's agentic gate table said inline accuracy is "per point — every point must clear", which reads as an accuracy run being required at every Pareto point. The checker requires one per mandatory region (accuracy-coverage) and gates whatever results are present. Say so in the README, agentic_targets and the inline gate's docstring. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 8 +++++++- src/submission_checker/agentic_targets.py | 10 ++++++---- src/submission_checker/models/aggregate/context.py | 6 +++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 494821b..67c85d0 100644 --- a/README.md +++ b/README.md @@ -529,6 +529,12 @@ keeps the Offline requirement in force rather than letting one mislabelled point it off. +Accuracy is **required** once per mandatory region, not at every point: +`accuracy-coverage` asks for one result in each of the four mandatory bands (plus the +Offline point for a non-agentic benchmark), and a band with several points needs only +one of them to carry accuracy. The gates below then judge whatever results a submission +includes, so every result present must clear. + Accuracy is gated differently too. §15's gate folds every dataset of a point into one sample-weighted score per metric; the agentic benchmarks gate three quantities that do not reduce that way, so for a recognised agentic model it stands down in favour of the @@ -536,7 +542,7 @@ three rules above: | Quantity | Source | Aggregation | |---|---|---| -| Inline accuracy | the `dataset_type: performance` entry in the accuracy results (named `performance` by the client) | per point — every point must clear | +| Inline accuracy | the `dataset_type: performance` entry in the accuracy results (named `performance` by the client) | each result on its own — every point that carries one must clear. Only one accuracy run per mandatory region is required (`accuracy-coverage`); points without one are not gated | | SWE-bench accuracy | `swe_bench` in the accuracy results | **mean-of-4** across the mandatory regions (§4.3's multi-turn branch) | | OSL per-turn mean | `output_sequence_lengths_full_run.output_sequence_lengths.avg` in `result_summary.json` | per point, against a range | diff --git a/src/submission_checker/agentic_targets.py b/src/submission_checker/agentic_targets.py index c4ffc75..bffd7d3 100644 --- a/src/submission_checker/agentic_targets.py +++ b/src/submission_checker/agentic_targets.py @@ -13,9 +13,11 @@ point into one sample-weighted score per metric and gates each point against it. The agentic benchmarks gate three quantities that do not reduce that way: -* **Inline accuracy** — per point, from the entry the client scores on the performance - run (``dataset_type: performance``; see - :meth:`~submission_checker.models.AccuracyResult.performance_dataset`). +* **Inline accuracy** — each result on its own, from the entry the client scores on + the performance run (``dataset_type: performance``; see + :meth:`~submission_checker.models.AccuracyResult.performance_dataset`). Every point + that carries one must clear; only one per mandatory region is required + (``accuracy-coverage``). * **SWE-bench accuracy** — **mean-of-N across points**, not per point: "average one SWE-bench accuracy result from each of the four mandatory regions (``N = 4``), then compare that mean with the model-specific SWE-bench threshold". This is §4.3's @@ -65,7 +67,7 @@ class AgenticTargets: #: Model name as §3.2 publishes it, for messages. name: str - #: Inline accuracy floor, per point, on a 0–100 scale. + #: Inline accuracy floor, for each point that reports one, on a 0–100 scale. inline_min: float | None #: SWE-bench floor for the mean of the N results, on a 0–100 scale. swebench_min: float | None diff --git a/src/submission_checker/models/aggregate/context.py b/src/submission_checker/models/aggregate/context.py index 3c3a16f..bc1697a 100644 --- a/src/submission_checker/models/aggregate/context.py +++ b/src/submission_checker/models/aggregate/context.py @@ -656,7 +656,11 @@ def _check_agentic_accuracy(self) -> ModelContext: return self def _gate_agentic_inline(self, targets: AgenticTargets) -> None: - """Inline accuracy, per point: "Every … submitted Pareto point must satisfy". + """Inline accuracy, at each point that reports it. + + "Every … submitted Pareto point must satisfy" applies to the results submitted. + Points without one are skipped; that one result per mandatory region exists is + ``accuracy-coverage``'s check. Read from the entry scored on the performance run, which the client names ``performance`` rather than after the performance dataset.