From 1a6380efafd2b39d7ea0310e2081c69f59b05f68 Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:30:07 -0500 Subject: [PATCH 1/6] Derive e2e_avg_interactivity from the client's summary totals The reference client never writes output_tokens_per_turn_total or e2e_turn_time_seconds_total. It reports the same two sums as output_sequence_lengths.total (tokens) and latency.total (ns of per-turn request latency), and stores e2e_avg_interactivity as their ratio only when no sample failed (endpoints metrics/report.py). So every client-written point, agentic or single-turn, failed agentic-metric-consistency with "reported, but ... no inputs to derive it from". PointSummary now declares latency and falls back to the client's totals when both named sums are absent and n_samples_failed == 0; the named sums still take precedence. The fail message and README row name both sources. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- src/submission_checker/data/messages.yaml | 5 +- .../models/file/point_summary.py | 12 ++++ .../test_checks_specdec_agentic.py | 56 +++++++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 873acd53..aca764b1 100644 --- a/README.md +++ b/README.md @@ -478,7 +478,7 @@ set published after this release. | `metric-consistency-tpot-p90` | §9.1 | Reported TPOT P90 present, finite, strictly positive | | `metric-consistency-tps-per-user` | §9.1 | Stored `tps_per_user` matches `1000 / tpot_p90_ms` | | `metric-consistency-tps-per-kw` | §4.5.3 | Stored `system_tps_per_kw` matches `system_tps / point_power_kw` — see [per-point normalisation](#per-point-normalisation-453) | -| `agentic-metric-consistency` | §4.1 | `e2e_avg_interactivity` is derivable from its reported inputs | +| `agentic-metric-consistency` | §4.1 | `e2e_avg_interactivity` is derivable from its reported inputs: the per-turn sums, or the client's `output_sequence_lengths.total / latency.total` when no sample failed | ### Accuracy (§15) diff --git a/src/submission_checker/data/messages.yaml b/src/submission_checker/data/messages.yaml index 46143a10..ccdc839b 100644 --- a/src/submission_checker/data/messages.yaml +++ b/src/submission_checker/data/messages.yaml @@ -108,8 +108,9 @@ agentic-metric-consistency: spec: §4.1 messages: pass: e2e_avg_interactivity={derived:.4f} tok/s across completed turns - fail: e2e_avg_interactivity {stored!r} is reported, but the summary states no output_tokens_per_turn_total - / e2e_turn_time_seconds_total to derive it from (§4.1) + fail: e2e_avg_interactivity {stored!r} is reported, but the summary states neither output_tokens_per_turn_total + / e2e_turn_time_seconds_total nor a failure-free output_sequence_lengths.total / latency.total to derive + it from (§4.1) fail-2: stored e2e_avg_interactivity {stored:.4f} ≠ derived {derived:.4f} (rel err {rel_err:.1%}) agentic-osl-range: title: Agentic OSL range diff --git a/src/submission_checker/models/file/point_summary.py b/src/submission_checker/models/file/point_summary.py index c768af7b..0c6b6c4f 100644 --- a/src/submission_checker/models/file/point_summary.py +++ b/src/submission_checker/models/file/point_summary.py @@ -65,6 +65,8 @@ class PointSummary(BaseModel): ttft: PercentileStats = Field(default_factory=PercentileStats) tpot: PercentileStats = Field(default_factory=PercentileStats) output_sequence_lengths: PercentileStats = Field(default_factory=PercentileStats) + #: Per-sample request latency in nanoseconds; ``total`` is its sum over the samples. + latency: PercentileStats = Field(default_factory=PercentileStats) #: §4.1 agentic inputs. Summed across every completed turn of every trajectory; #: e2e_turn_time excludes tool-call execution time. @@ -147,9 +149,19 @@ def e2e_avg_interactivity(self) -> float | None: The agentic analogue of ``tps_per_user`` — the output-token rate across completed turns, one scalar per measurement point. ``None`` for a single-turn benchmark, which reports neither input. + + The reference client reports the same two sums under its own names: + ``output_sequence_lengths.total`` (tokens) and ``latency.total`` (nanoseconds of + per-turn request latency, so tool-call time between turns is not in it). Those + are used when the named sums are absent, and only when no sample failed: output + tokens are recorded for successful turns only, latency for every terminal one, + which is why the client itself omits the ratio then. """ tokens = self.output_tokens_per_turn_total seconds = self.e2e_turn_time_seconds_total + if tokens is None and seconds is None and self.n_samples_failed == 0: + tokens = self.output_sequence_lengths.total + seconds = self.latency.total / 1e9 if tokens is None or seconds is None or seconds <= 0: return None return tokens / seconds diff --git a/tests/submission_checker/test_checks_specdec_agentic.py b/tests/submission_checker/test_checks_specdec_agentic.py index 6dd2ff3b..f99bc086 100644 --- a/tests/submission_checker/test_checks_specdec_agentic.py +++ b/tests/submission_checker/test_checks_specdec_agentic.py @@ -266,3 +266,59 @@ def test_zero_turn_time_is_not_a_division_error(self, tmp_path: Path) -> None: e2e_turn_time_seconds_total=0.0, ) assert summary.e2e_avg_interactivity is None + + # The reference client writes the same two sums under its own names, + # output_sequence_lengths.total (tokens) and latency.total (ns), and leaves + # e2e_avg_interactivity out when any sample failed (endpoints metrics/report.py). + + def test_derived_from_the_clients_token_and_latency_totals(self) -> None: + summary = PointSummary( + n_samples_completed=1, + duration_ns=1.0, + output_sequence_lengths={"total": 6000.0}, + latency={"total": 120_000_000_000.0}, + ) + assert summary.e2e_avg_interactivity == pytest.approx(50.0) + + def test_client_report_with_its_own_value_passes(self, tmp_path: Path) -> None: + """A measured client report: 30,048,888 tokens over 65,037.28 s of turn latency.""" + result = self._result( + tmp_path, + output_sequence_lengths={"total": 30048888}, + latency={"total": 65037277147776}, + e2e_avg_interactivity=462.02561542857495, + ) + hits = self._hits(result) + assert hits and all(r.severity == Severity.INFO for r in hits) + + def test_client_report_with_a_wrong_value_errors(self, tmp_path: Path) -> None: + result = self._result( + tmp_path, + output_sequence_lengths={"total": 6000.0}, + latency={"total": 120_000_000_000.0}, + e2e_avg_interactivity=999.0, + ) + errors = [r for r in self._hits(result) if r.severity == Severity.ERROR] + assert [r.key for r in errors] == ["fail-2"] # a mismatch, not missing inputs + + def test_client_totals_are_not_paired_when_a_sample_failed(self) -> None: + """Tokens cover successful turns only and latency every terminal one.""" + totals = { + "n_samples_completed": 1, + "duration_ns": 1.0, + "output_sequence_lengths": {"total": 6000.0}, + "latency": {"total": 120_000_000_000.0}, + } + assert PointSummary(**totals).e2e_avg_interactivity == pytest.approx(50.0) + assert PointSummary(**totals, n_samples_failed=1).e2e_avg_interactivity is None + + def test_the_named_sums_take_precedence_over_the_client_totals(self) -> None: + summary = PointSummary( + n_samples_completed=1, + duration_ns=1.0, + output_tokens_per_turn_total=6000.0, + e2e_turn_time_seconds_total=120.0, + output_sequence_lengths={"total": 1.0}, + latency={"total": 1_000_000_000.0}, + ) + assert summary.e2e_avg_interactivity == pytest.approx(50.0) From 5c9d6704d179a3060a2f08b09307eb5479d2993c Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:41:00 -0500 Subject: [PATCH 2/6] Ship the agentic reference README's approved drafters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §2.9.4's list for the agentic benchmarks is published in mlcommons/endpoints examples/10_Agentic_Inference/README.md ("Approved Checkpoints and Speculative-Decoding Heads"), but the bundled list was still empty, so every speculative-decoding point failed approved-drafter. Transcribe it: the two Kimi K3 DSpark heads, and the native heads of the DeepSeek-V4.1-Flash and three Qwen3.6-35B-A3B approved checkpoints. Heads with weights are weight-identified; weight_checksum is the pinned Hugging Face revision (git-sha1:), whose tree pins each weight file's SHA-256. Every entry's approved_cohort is 2026-09-C0, the cohort in which the README first published the list (endpoints#494, 2026-09-10); the DeepSeek-V4.1-Flash entry, added by endpoints#519 on 2026-09-30, is recorded against the same cohort. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 14 +++-- .../data/approved_drafters.yaml | 56 +++++++++++++++++-- .../test_checks_specdec_agentic.py | 44 ++++++++++++++- 3 files changed, 100 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index aca764b1..1c85307f 100644 --- a/README.md +++ b/README.md @@ -454,11 +454,15 @@ not satisfy High Concurrency coverage. | `drafter-approval-lead-time` | §2.9.4 | Approved at least two cohorts before `target_cohort` | | `drafter-list-registry` | §2.9.4 | Warns when the drafter list itself cannot be read | -The approved list ships as data (`src/submission_checker/data/approved_drafters.yaml`) -and is **empty** — §2.9.4's list has not been published yet, and an empty registry means -speculative decoding is not permitted for any benchmark, which is §2.9.4's own rule for a -benchmark with no approved drafter. Point `--approved-drafters FILE` or -`$MLPERF_ENDPOINTS_APPROVED_DRAFTERS` at a published list. +The approved list ships as data (`src/submission_checker/data/approved_drafters.yaml`), +transcribed from the agentic reference README's "Approved Checkpoints and +Speculative-Decoding Heads" (mlcommons/endpoints `examples/10_Agentic_Inference`): the +Kimi K3 DSpark heads, and the native heads of the approved DeepSeek-V4.1-Flash and +Qwen3.6-35B-A3B checkpoints. Each is weight-identified by its Hugging Face revision +(`weight_checksum: git-sha1:`), which a point's `speculative_decoding` block must +declare to match. A benchmark with no entry has no approved drafter, so speculative +decoding stays disallowed for it (§2.9.4). Point `--approved-drafters FILE` or +`$MLPERF_ENDPOINTS_APPROVED_DRAFTERS` at a newer published list. The published sets ship as data (`src/submission_checker/data/seed_sets.yaml`), mirrored from the policies repo's `seedset.yaml`. The file's `cohort-id` is the cohort its sets diff --git a/src/submission_checker/data/approved_drafters.yaml b/src/submission_checker/data/approved_drafters.yaml index 542b62b3..ad572b30 100644 --- a/src/submission_checker/data/approved_drafters.yaml +++ b/src/submission_checker/data/approved_drafters.yaml @@ -1,13 +1,14 @@ # Approved speculative-decoding drafters (§2.9.4), per benchmark. # # §2.9.4 makes this "a curated, published list, not a single fixed drafter", -# published in the reference repository and versioned per submission round. No such -# list has been published yet, so this file ships empty and the checker treats an -# empty registry as "speculative decoding is not yet permitted for any benchmark" — -# which is exactly §2.9.4's rule for a benchmark with no approved drafter. +# published in the reference repository and versioned per submission round. The +# agentic benchmarks' list is published in mlcommons/endpoints +# examples/10_Agentic_Inference/README.md, "Approved Checkpoints and +# Speculative-Decoding Heads"; the entries below transcribe it. A benchmark absent +# here still has no approved drafter, so speculative decoding stays disallowed for it. # # Override with `--approved-drafters FILE` or $MLPERF_ENDPOINTS_APPROVED_DRAFTERS -# once the task force publishes one, without waiting for a checker release. +# when the task force publishes a newer list, without waiting for a checker release. # # Shape, per §2.9.4's two identification forms: # @@ -21,5 +22,48 @@ # target_checksum: sha256:... # configuration-identified # configuration: # exit_layer: 24 +# +# How the reference entries are transcribed: +# +# - Weight-identified. MTP and DSpark heads carry their own weights (§2.9.4's +# configuration form is for drafters with none). A head native to an approved +# checkpoint is identified by that checkpoint. +# - weight_checksum is the Hugging Face revision the README pins, as +# `git-sha1:`. A revision is the git commit of the model repository; its +# tree pins every weight file's SHA-256 through the LFS pointers, so it identifies +# the published weights exactly. +# - approved_cohort is 2026-09-C0 for every entry: the cohort in which the README first +# published the approved list (endpoints#494, 2026-09-10, after the 2026-09-02 C0 +# publication). The DeepSeek-V4.1-Flash entry was added to that list later +# (endpoints#519, 2026-09-30) and is recorded against the same cohort. + +drafters: + # Kimi K3: separately published DSpark heads (endpoints#494). + - benchmark: kimi-k3 + approved_cohort: 2026-09-C0 + model_id: RadixArk/Kimi-K3-DSpark + weight_checksum: git-sha1:3c5bac301d9cf392706189d82ed947feca6c2f0f + - benchmark: kimi-k3 + approved_cohort: 2026-09-C0 + model_id: Inferact/Kimi-K3-DSpark + weight_checksum: git-sha1:cf6b8244620e7ea4b0651d214f28e89eac75bed6 + + # DeepSeek-V4.1-Flash: the native DSpark head in the approved checkpoint (endpoints#519). + - benchmark: deepseek-v4_1-flash + approved_cohort: 2026-09-C0 + model_id: deepseek-ai/DeepSeek-V4.1-Flash + weight_checksum: git-sha1:dba1be0a40aa45a94ad051997016db3960a90277 -drafters: [] + # Qwen3.6-35B-A3B: the native MTP heads in each approved checkpoint (endpoints#494). + - benchmark: qwen3_6-35b-a3b + approved_cohort: 2026-09-C0 + model_id: Qwen/Qwen3.6-35B-A3B + weight_checksum: git-sha1:995ad96eacd98c81ed38be0c5b274b04031597b0 + - benchmark: qwen3_6-35b-a3b + approved_cohort: 2026-09-C0 + model_id: Qwen/Qwen3.6-35B-A3B-FP8 + weight_checksum: git-sha1:95a723d08a9490559dae23d0cff1d9466213d989 + - benchmark: qwen3_6-35b-a3b + approved_cohort: 2026-09-C0 + model_id: nvidia/Qwen3.6-35B-A3B-NVFP4 + weight_checksum: git-sha1:1355db6a052410cfd62085d94b58866fd0f2c3c5 diff --git a/tests/submission_checker/test_checks_specdec_agentic.py b/tests/submission_checker/test_checks_specdec_agentic.py index f99bc086..6ccf9259 100644 --- a/tests/submission_checker/test_checks_specdec_agentic.py +++ b/tests/submission_checker/test_checks_specdec_agentic.py @@ -9,6 +9,7 @@ import pytest import yaml +from submission_checker.cohorts import Cohort from submission_checker.drafters import ( ApprovedDrafter, DrafterListError, @@ -69,10 +70,47 @@ def _errors(binding: DrafterBinding, rule: str) -> list: @pytest.mark.unit class TestDrafterList: - def test_bundled_list_ships_empty(self) -> None: - """§2.9.4's list is not published yet; empty means "none approved".""" + #: The approved heads in mlcommons/endpoints examples/10_Agentic_Inference/README.md, + #: "Approved Checkpoints and Speculative-Decoding Heads": benchmark -> {model_id: revision}. + #: Qwen and DeepSeek-V4.1-Flash heads are native to their approved checkpoints. + _REFERENCE_HEADS = { + "kimi-k3": { + "RadixArk/Kimi-K3-DSpark": "3c5bac301d9cf392706189d82ed947feca6c2f0f", + "Inferact/Kimi-K3-DSpark": "cf6b8244620e7ea4b0651d214f28e89eac75bed6", + }, + "deepseek-v4_1-flash": { + "deepseek-ai/DeepSeek-V4.1-Flash": "dba1be0a40aa45a94ad051997016db3960a90277", + }, + "qwen3_6-35b-a3b": { + "Qwen/Qwen3.6-35B-A3B": "995ad96eacd98c81ed38be0c5b274b04031597b0", + "Qwen/Qwen3.6-35B-A3B-FP8": "95a723d08a9490559dae23d0cff1d9466213d989", + "nvidia/Qwen3.6-35B-A3B-NVFP4": "1355db6a052410cfd62085d94b58866fd0f2c3c5", + }, + } + + def test_bundled_list_carries_the_reference_approved_heads(self) -> None: assert bundled_drafters_path().is_file() - assert load_approved_drafters() == {} + bundled = { + benchmark: {d.model_id: d.weight_checksum for d in drafters} + for benchmark, drafters in load_approved_drafters().items() + } + assert bundled == { + benchmark: {model_id: f"git-sha1:{rev}" for model_id, rev in heads.items()} + for benchmark, heads in self._REFERENCE_HEADS.items() + } + + def test_bundled_heads_are_recorded_against_the_lists_first_cohort(self) -> None: + """Every entry counts from 2026-09-C0, when the README first published the list.""" + cohorts = { + d.model_id: d.approved_cohort for ds in load_approved_drafters().values() for d in ds + } + assert set(cohorts.values()) == {"2026-09-C0"} + + def test_bundled_heads_are_usable_in_the_2026_10_c1_cohort(self) -> None: + target = Cohort.parse("2026-10-C1") + for drafters in load_approved_drafters().values(): + for drafter in drafters: + assert drafter.earliest_target_cohort() <= target, drafter.model_id def test_entries_group_by_benchmark(self, tmp_path: Path) -> None: path = tmp_path / "d.yaml" From 223f45c7014c3a1a3ec040b2576eac45749f3c49 Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:55:09 -0500 Subject: [PATCH 3/6] Respect each drafter approval cohort and clarify metric units --- README.md | 2 +- .../data/approved_drafters.yaml | 10 +++---- .../models/file/point_summary.py | 4 +-- .../test_checks_specdec_agentic.py | 30 +++++++++++++++++-- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 1c85307f..468fc70c 100644 --- a/README.md +++ b/README.md @@ -482,7 +482,7 @@ set published after this release. | `metric-consistency-tpot-p90` | §9.1 | Reported TPOT P90 present, finite, strictly positive | | `metric-consistency-tps-per-user` | §9.1 | Stored `tps_per_user` matches `1000 / tpot_p90_ms` | | `metric-consistency-tps-per-kw` | §4.5.3 | Stored `system_tps_per_kw` matches `system_tps / point_power_kw` — see [per-point normalisation](#per-point-normalisation-453) | -| `agentic-metric-consistency` | §4.1 | `e2e_avg_interactivity` is derivable from its reported inputs: the per-turn sums, or the client's `output_sequence_lengths.total / latency.total` when no sample failed | +| `agentic-metric-consistency` | §4.1 | `e2e_avg_interactivity` is derivable from its reported inputs: the per-turn sums, or the client's `output_sequence_lengths.total / (latency.total / 1e9)` when no sample failed (latency is in nanoseconds) | ### Accuracy (§15) diff --git a/src/submission_checker/data/approved_drafters.yaml b/src/submission_checker/data/approved_drafters.yaml index ad572b30..04206237 100644 --- a/src/submission_checker/data/approved_drafters.yaml +++ b/src/submission_checker/data/approved_drafters.yaml @@ -32,10 +32,10 @@ # `git-sha1:`. A revision is the git commit of the model repository; its # tree pins every weight file's SHA-256 through the LFS pointers, so it identifies # the published weights exactly. -# - approved_cohort is 2026-09-C0 for every entry: the cohort in which the README first -# published the approved list (endpoints#494, 2026-09-10, after the 2026-09-02 C0 -# publication). The DeepSeek-V4.1-Flash entry was added to that list later -# (endpoints#519, 2026-09-30) and is recorded against the same cohort. +# - approved_cohort follows each entry's publication: Kimi and Qwen were published +# in endpoints#494 on 2026-09-10, in 2026-09-C0. DeepSeek-V4.1-Flash was added in +# endpoints#519 on 2026-09-30, in 2026-09-C1 (after the 2026-09-16 C1 publication). +# The later addition must satisfy its own two-cohort approval lead time. drafters: # Kimi K3: separately published DSpark heads (endpoints#494). @@ -50,7 +50,7 @@ drafters: # DeepSeek-V4.1-Flash: the native DSpark head in the approved checkpoint (endpoints#519). - benchmark: deepseek-v4_1-flash - approved_cohort: 2026-09-C0 + approved_cohort: 2026-09-C1 model_id: deepseek-ai/DeepSeek-V4.1-Flash weight_checksum: git-sha1:dba1be0a40aa45a94ad051997016db3960a90277 diff --git a/src/submission_checker/models/file/point_summary.py b/src/submission_checker/models/file/point_summary.py index 0c6b6c4f..e263b0f8 100644 --- a/src/submission_checker/models/file/point_summary.py +++ b/src/submission_checker/models/file/point_summary.py @@ -147,8 +147,8 @@ def e2e_avg_interactivity(self) -> float | None: """§4.1: ``sum(output_tokens_per_turn) / sum(e2e_turn_time_seconds)``. The agentic analogue of ``tps_per_user`` — the output-token rate across - completed turns, one scalar per measurement point. ``None`` for a single-turn - benchmark, which reports neither input. + completed turns, one scalar per measurement point. Also derivable for + single-turn reports; ``None`` when usable input totals are unavailable. The reference client reports the same two sums under its own names: ``output_sequence_lengths.total`` (tokens) and ``latency.total`` (nanoseconds of diff --git a/tests/submission_checker/test_checks_specdec_agentic.py b/tests/submission_checker/test_checks_specdec_agentic.py index 6ccf9259..d3d8611a 100644 --- a/tests/submission_checker/test_checks_specdec_agentic.py +++ b/tests/submission_checker/test_checks_specdec_agentic.py @@ -99,12 +99,36 @@ def test_bundled_list_carries_the_reference_approved_heads(self) -> None: for benchmark, heads in self._REFERENCE_HEADS.items() } - def test_bundled_heads_are_recorded_against_the_lists_first_cohort(self) -> None: - """Every entry counts from 2026-09-C0, when the README first published the list.""" + def test_bundled_heads_record_their_own_publication_cohort(self) -> None: + """The September 30 DeepSeek addition cannot inherit the September 10 approval.""" cohorts = { d.model_id: d.approved_cohort for ds in load_approved_drafters().values() for d in ds } - assert set(cohorts.values()) == {"2026-09-C0"} + assert cohorts == { + model_id: "2026-09-C1" if benchmark == "deepseek-v4_1-flash" else "2026-09-C0" + for benchmark, heads in self._REFERENCE_HEADS.items() + for model_id in heads + } + + @pytest.mark.parametrize( + ("target_cohort", "expected_severity"), + [("2026-10-C0", Severity.ERROR), ("2026-10-C1", Severity.INFO)], + ) + def test_deepseek_bundled_head_respects_its_approval_lead_time( + self, tmp_path: Path, target_cohort: str, expected_severity: Severity + ) -> None: + benchmark = "deepseek-v4_1-flash" + approved = load_approved_drafters()[benchmark] + binding = DrafterBinding( + points=[ + _point(tmp_path, {"weight_checksum": approved[0].weight_checksum}, target_cohort) + ], + approved=approved, + benchmark=benchmark, + model_dir=tmp_path, + ) + results = [r for r in binding._check_results if r.rule == "drafter-approval-lead-time"] + assert [r.severity for r in results] == [expected_severity] def test_bundled_heads_are_usable_in_the_2026_10_c1_cohort(self) -> None: target = Cohort.parse("2026-10-C1") From d61609811a355d0ecb837553c596ea894725578f Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:06:30 -0500 Subject: [PATCH 4/6] Record initial drafter approvals in the 2026-10-C1 cohort --- README.md | 6 +++ .../data/approved_drafters.yaml | 20 ++++---- .../test_checks_specdec_agentic.py | 49 +++++++++---------- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 468fc70c..cd6cc1b7 100644 --- a/README.md +++ b/README.md @@ -464,6 +464,12 @@ declare to match. A benchmark with no entry has no approved drafter, so speculat decoding stays disallowed for it (§2.9.4). Point `--approved-drafters FILE` or `$MLPERF_ENDPOINTS_APPROVED_DRAFTERS` at a newer published list. +All six initial drafter approvals are recorded against `2026-10-C1`, matching the +published seed-set cohort. The two-cohort approval lead time makes their earliest +eligible target cohort `2026-11-C1`. +Approval cohorts are recorded per drafter, so future additions can carry later +cohorts in the same registry. + The published sets ship as data (`src/submission_checker/data/seed_sets.yaml`), mirrored from the policies repo's `seedset.yaml`. The file's `cohort-id` is the cohort its sets were published for; §4.6's four-cohort adoption window is derived from it. Point diff --git a/src/submission_checker/data/approved_drafters.yaml b/src/submission_checker/data/approved_drafters.yaml index 04206237..1f7cc82b 100644 --- a/src/submission_checker/data/approved_drafters.yaml +++ b/src/submission_checker/data/approved_drafters.yaml @@ -32,38 +32,38 @@ # `git-sha1:`. A revision is the git commit of the model repository; its # tree pins every weight file's SHA-256 through the LFS pointers, so it identifies # the published weights exactly. -# - approved_cohort follows each entry's publication: Kimi and Qwen were published -# in endpoints#494 on 2026-09-10, in 2026-09-C0. DeepSeek-V4.1-Flash was added in -# endpoints#519 on 2026-09-30, in 2026-09-C1 (after the 2026-09-16 C1 publication). -# The later addition must satisfy its own two-cohort approval lead time. +# - These initial entries are recorded against 2026-10-C1, matching the published +# cohort in seed_sets.yaml. The two-cohort approval lead time therefore makes +# them eligible starting with target cohort 2026-11-C1. +# Future entries record their own approval cohorts alongside these entries. drafters: # Kimi K3: separately published DSpark heads (endpoints#494). - benchmark: kimi-k3 - approved_cohort: 2026-09-C0 + approved_cohort: 2026-10-C1 model_id: RadixArk/Kimi-K3-DSpark weight_checksum: git-sha1:3c5bac301d9cf392706189d82ed947feca6c2f0f - benchmark: kimi-k3 - approved_cohort: 2026-09-C0 + approved_cohort: 2026-10-C1 model_id: Inferact/Kimi-K3-DSpark weight_checksum: git-sha1:cf6b8244620e7ea4b0651d214f28e89eac75bed6 # DeepSeek-V4.1-Flash: the native DSpark head in the approved checkpoint (endpoints#519). - benchmark: deepseek-v4_1-flash - approved_cohort: 2026-09-C1 + approved_cohort: 2026-10-C1 model_id: deepseek-ai/DeepSeek-V4.1-Flash weight_checksum: git-sha1:dba1be0a40aa45a94ad051997016db3960a90277 # Qwen3.6-35B-A3B: the native MTP heads in each approved checkpoint (endpoints#494). - benchmark: qwen3_6-35b-a3b - approved_cohort: 2026-09-C0 + approved_cohort: 2026-10-C1 model_id: Qwen/Qwen3.6-35B-A3B weight_checksum: git-sha1:995ad96eacd98c81ed38be0c5b274b04031597b0 - benchmark: qwen3_6-35b-a3b - approved_cohort: 2026-09-C0 + approved_cohort: 2026-10-C1 model_id: Qwen/Qwen3.6-35B-A3B-FP8 weight_checksum: git-sha1:95a723d08a9490559dae23d0cff1d9466213d989 - benchmark: qwen3_6-35b-a3b - approved_cohort: 2026-09-C0 + approved_cohort: 2026-10-C1 model_id: nvidia/Qwen3.6-35B-A3B-NVFP4 weight_checksum: git-sha1:1355db6a052410cfd62085d94b58866fd0f2c3c5 diff --git a/tests/submission_checker/test_checks_specdec_agentic.py b/tests/submission_checker/test_checks_specdec_agentic.py index d3d8611a..fb1fd963 100644 --- a/tests/submission_checker/test_checks_specdec_agentic.py +++ b/tests/submission_checker/test_checks_specdec_agentic.py @@ -9,7 +9,6 @@ import pytest import yaml -from submission_checker.cohorts import Cohort from submission_checker.drafters import ( ApprovedDrafter, DrafterListError, @@ -99,42 +98,42 @@ def test_bundled_list_carries_the_reference_approved_heads(self) -> None: for benchmark, heads in self._REFERENCE_HEADS.items() } - def test_bundled_heads_record_their_own_publication_cohort(self) -> None: - """The September 30 DeepSeek addition cannot inherit the September 10 approval.""" + def test_bundled_heads_record_the_initial_2026_10_c1_cohort(self) -> None: + """Initial approvals use the cohort published alongside the seed sets.""" cohorts = { d.model_id: d.approved_cohort for ds in load_approved_drafters().values() for d in ds } assert cohorts == { - model_id: "2026-09-C1" if benchmark == "deepseek-v4_1-flash" else "2026-09-C0" - for benchmark, heads in self._REFERENCE_HEADS.items() + model_id: "2026-10-C1" + for heads in self._REFERENCE_HEADS.values() for model_id in heads } @pytest.mark.parametrize( ("target_cohort", "expected_severity"), - [("2026-10-C0", Severity.ERROR), ("2026-10-C1", Severity.INFO)], + [ + ("2026-10-C1", Severity.ERROR), + ("2026-11-C0", Severity.ERROR), + ("2026-11-C1", Severity.INFO), + ], ) - def test_deepseek_bundled_head_respects_its_approval_lead_time( + def test_bundled_heads_respect_their_approval_lead_time( self, tmp_path: Path, target_cohort: str, expected_severity: Severity ) -> None: - benchmark = "deepseek-v4_1-flash" - approved = load_approved_drafters()[benchmark] - binding = DrafterBinding( - points=[ - _point(tmp_path, {"weight_checksum": approved[0].weight_checksum}, target_cohort) - ], - approved=approved, - benchmark=benchmark, - model_dir=tmp_path, - ) - results = [r for r in binding._check_results if r.rule == "drafter-approval-lead-time"] - assert [r.severity for r in results] == [expected_severity] - - def test_bundled_heads_are_usable_in_the_2026_10_c1_cohort(self) -> None: - target = Cohort.parse("2026-10-C1") - for drafters in load_approved_drafters().values(): - for drafter in drafters: - assert drafter.earliest_target_cohort() <= target, drafter.model_id + for benchmark, approved in load_approved_drafters().items(): + for drafter in approved: + binding = DrafterBinding( + points=[ + _point(tmp_path, {"weight_checksum": drafter.weight_checksum}, target_cohort) + ], + approved=approved, + benchmark=benchmark, + model_dir=tmp_path, + ) + results = [ + r for r in binding._check_results if r.rule == "drafter-approval-lead-time" + ] + assert [r.severity for r in results] == [expected_severity], drafter.model_id def test_entries_group_by_benchmark(self, tmp_path: Path) -> None: path = tmp_path / "d.yaml" From 3e6d275609a01f1b954cc9b5de42a6077b79b23a Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:36:17 -0500 Subject: [PATCH 5/6] Record the initial drafter approvals in 2026-09-C1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §2.9.4 records an approval against the cohort in which the updated list is published. The reference README published the Qwen and Kimi heads on 2026-09-10 (endpoints#494) and the DeepSeek-V4.1-Flash head on 2026-09-30 (endpoints#519), so all six are recorded in 2026-09-C1, the cohort window by which both were published (counted from the next publication date instead, the DeepSeek-V4.1-Flash entry would be 2026-10-C0). With the two-cohort lead time they are eligible from target cohort 2026-10-C1. Tests cover the approval cohort, one cohort too early, and the first eligible cohort. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 9 +++++--- .../data/approved_drafters.yaml | 23 +++++++++++-------- .../test_checks_specdec_agentic.py | 16 ++++++++----- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index cd6cc1b7..b1e873a3 100644 --- a/README.md +++ b/README.md @@ -464,9 +464,12 @@ declare to match. A benchmark with no entry has no approved drafter, so speculat decoding stays disallowed for it (§2.9.4). Point `--approved-drafters FILE` or `$MLPERF_ENDPOINTS_APPROVED_DRAFTERS` at a newer published list. -All six initial drafter approvals are recorded against `2026-10-C1`, matching the -published seed-set cohort. The two-cohort approval lead time makes their earliest -eligible target cohort `2026-11-C1`. +All six initial drafter approvals are recorded against `2026-09-C1`: the reference +README published them on 2026-09-10 (endpoints#494) and 2026-09-30 (endpoints#519), before +the `2026-10-C0` publication, and §2.9.4 records an approval against the cohort in which the +list is published. (Counted from the next publication date instead, the DeepSeek-V4.1-Flash +entry would be `2026-10-C0`.) The two-cohort approval lead time makes their earliest eligible +target cohort `2026-10-C1`. Approval cohorts are recorded per drafter, so future additions can carry later cohorts in the same registry. diff --git a/src/submission_checker/data/approved_drafters.yaml b/src/submission_checker/data/approved_drafters.yaml index 1f7cc82b..50f387ce 100644 --- a/src/submission_checker/data/approved_drafters.yaml +++ b/src/submission_checker/data/approved_drafters.yaml @@ -32,38 +32,43 @@ # `git-sha1:`. A revision is the git commit of the model repository; its # tree pins every weight file's SHA-256 through the LFS pointers, so it identifies # the published weights exactly. -# - These initial entries are recorded against 2026-10-C1, matching the published -# cohort in seed_sets.yaml. The two-cohort approval lead time therefore makes -# them eligible starting with target cohort 2026-11-C1. +# - These initial entries are recorded against 2026-09-C1. §2.9.4 records an approval +# against the cohort in which the updated list is published. The README published the +# Qwen and Kimi heads on 2026-09-10 (endpoints#494), after the 2026-09-02 C0 +# publication, and the DeepSeek-V4.1-Flash head on 2026-09-30 (endpoints#519), after the +# 2026-09-16 C1 one; 2026-09-C1 is the cohort window by which both were published. +# Counted instead from the next publication date, the DeepSeek-V4.1-Flash entry would be +# 2026-10-C0. The two-cohort approval lead time makes these entries eligible starting +# with target cohort 2026-10-C1. # Future entries record their own approval cohorts alongside these entries. drafters: # Kimi K3: separately published DSpark heads (endpoints#494). - benchmark: kimi-k3 - approved_cohort: 2026-10-C1 + approved_cohort: 2026-09-C1 model_id: RadixArk/Kimi-K3-DSpark weight_checksum: git-sha1:3c5bac301d9cf392706189d82ed947feca6c2f0f - benchmark: kimi-k3 - approved_cohort: 2026-10-C1 + approved_cohort: 2026-09-C1 model_id: Inferact/Kimi-K3-DSpark weight_checksum: git-sha1:cf6b8244620e7ea4b0651d214f28e89eac75bed6 # DeepSeek-V4.1-Flash: the native DSpark head in the approved checkpoint (endpoints#519). - benchmark: deepseek-v4_1-flash - approved_cohort: 2026-10-C1 + approved_cohort: 2026-09-C1 model_id: deepseek-ai/DeepSeek-V4.1-Flash weight_checksum: git-sha1:dba1be0a40aa45a94ad051997016db3960a90277 # Qwen3.6-35B-A3B: the native MTP heads in each approved checkpoint (endpoints#494). - benchmark: qwen3_6-35b-a3b - approved_cohort: 2026-10-C1 + approved_cohort: 2026-09-C1 model_id: Qwen/Qwen3.6-35B-A3B weight_checksum: git-sha1:995ad96eacd98c81ed38be0c5b274b04031597b0 - benchmark: qwen3_6-35b-a3b - approved_cohort: 2026-10-C1 + approved_cohort: 2026-09-C1 model_id: Qwen/Qwen3.6-35B-A3B-FP8 weight_checksum: git-sha1:95a723d08a9490559dae23d0cff1d9466213d989 - benchmark: qwen3_6-35b-a3b - approved_cohort: 2026-10-C1 + approved_cohort: 2026-09-C1 model_id: nvidia/Qwen3.6-35B-A3B-NVFP4 weight_checksum: git-sha1:1355db6a052410cfd62085d94b58866fd0f2c3c5 diff --git a/tests/submission_checker/test_checks_specdec_agentic.py b/tests/submission_checker/test_checks_specdec_agentic.py index fb1fd963..3e79c0ef 100644 --- a/tests/submission_checker/test_checks_specdec_agentic.py +++ b/tests/submission_checker/test_checks_specdec_agentic.py @@ -98,13 +98,17 @@ def test_bundled_list_carries_the_reference_approved_heads(self) -> None: for benchmark, heads in self._REFERENCE_HEADS.items() } - def test_bundled_heads_record_the_initial_2026_10_c1_cohort(self) -> None: - """Initial approvals use the cohort published alongside the seed sets.""" + def test_bundled_heads_record_the_2026_09_c1_cohort(self) -> None: + """The README published every head before the 2026-10-C0 publication. + + Qwen and Kimi on 2026-09-10 (endpoints#494), DeepSeek-V4.1-Flash on 2026-09-30 + (endpoints#519): no earlier than each entry's own publication cohort. + """ cohorts = { d.model_id: d.approved_cohort for ds in load_approved_drafters().values() for d in ds } assert cohorts == { - model_id: "2026-10-C1" + model_id: "2026-09-C1" for heads in self._REFERENCE_HEADS.values() for model_id in heads } @@ -112,9 +116,9 @@ def test_bundled_heads_record_the_initial_2026_10_c1_cohort(self) -> None: @pytest.mark.parametrize( ("target_cohort", "expected_severity"), [ - ("2026-10-C1", Severity.ERROR), - ("2026-11-C0", Severity.ERROR), - ("2026-11-C1", Severity.INFO), + ("2026-09-C1", Severity.ERROR), + ("2026-10-C0", Severity.ERROR), + ("2026-10-C1", Severity.INFO), ], ) def test_bundled_heads_respect_their_approval_lead_time( From aee08d7514adb565b06b25b964182c857c8f3aa0 Mon Sep 17 00:00:00 2001 From: arekay-nv <230885705+arekay-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:57:13 -0500 Subject: [PATCH 6/6] Keep the accuracy results a --mode both run carries submissions create assembles the bundle by filing each run as performance or accuracy from config.yaml datasets[0].type. A --mode both run lists the performance dataset first, so it is filed as performance, and accuracy_results.json was only written from a separate accuracy run. Its own accuracy/accuracy_results.json was dropped, so every --mode both submission failed accuracy-present and accuracy-coverage. When a concurrency has no separate accuracy run, use the performance run's accuracy/ directory if it has one. A separate accuracy run still wins, and a performance-only run's top-level results.json, its request log, is never taken for accuracy. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../submissions/builder.py | 10 +++ .../submissions/test_builder.py | 75 +++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/src/endpoints_submission_cli/submissions/builder.py b/src/endpoints_submission_cli/submissions/builder.py index 11fc3acb..715c4d36 100644 --- a/src/endpoints_submission_cli/submissions/builder.py +++ b/src/endpoints_submission_cli/submissions/builder.py @@ -602,10 +602,20 @@ def _write_point_dirs( _write_point_system_desc(point_dir, primary, max_tps) _write_point_extra_files(point_dir, primary, max_tps) + # A `--mode both` run is filed as performance but also carries accuracy/; use it + # when no separate accuracy run exists. Its top-level results.json is the + # performance request log, so only accuracy/ counts. + if accuracy_run is None and _carries_accuracy(primary): + accuracy_run = primary if accuracy_run is not None: _write_accuracy_results(point_dir, accuracy_run) +def _carries_accuracy(run: dict[str, Any]) -> bool: + """Whether a run's archive holds an ``accuracy/`` phase directory.""" + return any(rel.startswith(f"{layout.ACCURACY_SUBDIR}/") for rel in run.get("_extra_files", {})) + + #: Marker delimiting the block the builder appends to a copied point.yaml. _INJECTION_HEADER = "# --- added by endpoints-submission-cli: bundle-internal paths ---" diff --git a/tests/endpoints_submission_cli/submissions/test_builder.py b/tests/endpoints_submission_cli/submissions/test_builder.py index 672a47a1..359787f5 100644 --- a/tests/endpoints_submission_cli/submissions/test_builder.py +++ b/tests/endpoints_submission_cli/submissions/test_builder.py @@ -733,6 +733,81 @@ def test_endpoints_accuracy_artifact_is_used(self, run_folder: Path, tmp_path: P written = next(sub_dir.rglob("accuracy_results.json")) assert json.loads(written.read_text())["cnn_dailymail"]["score"]["rouge1"] == 42.0 + # `--mode both` writes performance/ and accuracy/ into one run. Its datasets list the + # performance dataset first, so the run is filed as performance; its accuracy/ must + # still reach the point. + + def _both_mode_archive( + self, run_folder: Path, tmp_path: Path, name: str, *, accuracy: dict | None + ) -> Path: + import shutil + + folder = tmp_path / name + shutil.copytree(run_folder, folder) + config = yaml.safe_load((folder / "config.yaml").read_text()) + config["datasets"] = [ + {"name": "cnn_dailymail", "type": "performance"}, + *([{"name": "cnn_dailymail_acc", "type": "accuracy"}] if accuracy else []), + ] + (folder / "config.yaml").write_text(yaml.dump(config)) + if accuracy is not None: + (folder / "accuracy").mkdir(exist_ok=True) + (folder / "accuracy" / "accuracy_results.json").write_text(json.dumps(accuracy)) + archive = tmp_path / f"{name}.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(folder, arcname=name) + return archive + + def test_mode_both_run_keeps_its_accuracy(self, run_folder: Path, tmp_path: Path) -> None: + accuracy = {"cnn_dailymail_acc": {"num_samples": 10, "score": {"rouge1": 41.0}}} + archive = self._both_mode_archive(run_folder, tmp_path, "both", accuracy=accuracy) + + sub_dir = build_submission_folder( + [("run-001", archive)], "standardized", "available", tmp_path / "sub" + ) + + written = next(sub_dir.rglob("accuracy_results.json")) + assert json.loads(written.read_text()) == accuracy + + def test_performance_log_is_not_taken_for_accuracy( + self, run_folder: Path, tmp_path: Path + ) -> None: + """A performance-only run's top-level results.json is its request log.""" + archive = self._both_mode_archive(run_folder, tmp_path, "perf", accuracy=None) + + sub_dir = build_submission_folder( + [("run-001", archive)], "standardized", "available", tmp_path / "sub" + ) + + assert not list(sub_dir.rglob("accuracy_results.json")) + + def test_a_separate_accuracy_run_still_wins(self, run_folder: Path, tmp_path: Path) -> None: + both = self._both_mode_archive( + run_folder, tmp_path, "both", accuracy={"from": {"num_samples": 1, "score": 1.0}} + ) + acc_only = self._both_mode_archive( + run_folder, tmp_path, "acc", accuracy={"from": {"num_samples": 1, "score": 2.0}} + ) + acc_dir = tmp_path / "acc-config" + with tarfile.open(acc_only) as tar: + tar.extractall(acc_dir, filter="data") + config_path = acc_dir / "acc" / "config.yaml" + config = yaml.safe_load(config_path.read_text()) + config["datasets"] = [{"name": "cnn_dailymail_acc", "type": "accuracy"}] + config_path.write_text(yaml.dump(config)) + with tarfile.open(acc_only, "w:gz") as tar: + tar.add(acc_dir / "acc", arcname="acc") + + sub_dir = build_submission_folder( + [("run-001", both), ("run-002", acc_only)], + "standardized", + "available", + tmp_path / "sub", + ) + + written = next(sub_dir.rglob("accuracy_results.json")) + assert json.loads(written.read_text())["from"]["score"] == 2.0 + @pytest.mark.unit class TestDirectoryNamingPrecedence: