Merge pull request #116 from arekay-nv/drafters/legacy-dsr1-gptoss-heads

Add the legacy DeepSeek-R1 and GPT-OSS-120B drafters

Author: arav-agarwal2

README.md

@@ -458,7 +458,9 @@ The approved list ships as data (`src/submission_checker/data/approved_drafters.
 transcribed from the agentic reference README's "Approved Checkpoints and
 Speculative-Decoding Heads" (mlcommons/endpoints `examples/10_Agentic_Inference`): the
 Kimi K3 DSpark heads, and the native heads of the approved DeepSeek-V4.1-Flash and
-Qwen3.6-35B-A3B checkpoints. Each is weight-identified by its Hugging Face revision
+Qwen3.6-35B-A3B checkpoints. It also carries the native MTP heads of the approved
+DeepSeek-R1 checkpoints and the GPT-OSS-120B EAGLE3 head, carried over from legacy
+submissions. Each is weight-identified by its Hugging Face revision
 (`weight_checksum: git-sha1:<revision>`), which a point's `speculative_decoding` block must
 declare to match. A benchmark with no entry has no approved drafter, so speculative
 decoding stays disallowed for it (§2.9.4). Point `--approved-drafters FILE` or
@@ -469,7 +471,9 @@ README published them on 2026-09-10 (endpoints#494) and 2026-09-30 (endpoints#51
 the `2026-10-C0` publication, and §2.9.4 records an approval against the cohort in which the
 list is published. (Counted from the next publication date instead, the DeepSeek-V4.1-Flash
 entry would be `2026-10-C0`.) The two-cohort approval lead time makes their earliest eligible
-target cohort `2026-10-C1`.
+target cohort `2026-10-C1`. The DeepSeek-R1 and GPT-OSS-120B heads, carried over from
+legacy submissions, are recorded with them in `2026-09-C1`, so they are also eligible from
+`2026-10-C1`.
 Approval cohorts are recorded per drafter, so future additions can carry later
 cohorts in the same registry.
 

src/submission_checker/data/approved_drafters.yaml

@@ -4,7 +4,8 @@
 # published in the reference repository and versioned per submission round. The
 # agentic benchmarks' list is published in mlcommons/endpoints
 # examples/10_Agentic_Inference/README.md, "Approved Checkpoints and
-# Speculative-Decoding Heads"; the entries below transcribe it. A benchmark absent
+# Speculative-Decoding Heads"; the entries below transcribe it. The DeepSeek-R1 and
+# gpt-oss-120b entries are carried over from legacy submissions. A benchmark absent
 # here still has no approved drafter, so speculative decoding stays disallowed for it.
 #
 # Override with `--approved-drafters FILE` or $MLPERF_ENDPOINTS_APPROVED_DRAFTERS
@@ -41,6 +42,11 @@
 #   2026-10-C0. The two-cohort approval lead time makes these entries eligible starting
 #   with target cohort 2026-10-C1.
 #   Future entries record their own approval cohorts alongside these entries.
+# - The DeepSeek-R1 and gpt-oss-120b entries are carried over from legacy submissions
+#   as task-force seed entries (§2.9.4: "the benchmark task force seeds the list with
+#   ≥ 1 approved head/drafter per benchmark model"). They are recorded with the other
+#   initial approvals in 2026-09-C1, so they are eligible starting with target cohort
+#   2026-10-C1.
 
 drafters:
   # Kimi K3: separately published DSpark heads (endpoints#494).
@@ -72,3 +78,23 @@ drafters:
     approved_cohort: 2026-09-C1
     model_id: nvidia/Qwen3.6-35B-A3B-NVFP4
     weight_checksum: git-sha1:1355db6a052410cfd62085d94b58866fd0f2c3c5
+
+  # DeepSeek-R1: the native MTP head (layer 61) in each approved checkpoint, carried
+  # over from legacy submissions. The NVFP4 checkpoint is post-training quantized on the
+  # MLPerf Inference deepseek-r1 calibration set and keeps the MTP layer.
+  - benchmark: deepseek-r1
+    approved_cohort: 2026-09-C1
+    model_id: deepseek-ai/DeepSeek-R1
+    weight_checksum: git-sha1:56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad
+  - benchmark: deepseek-r1
+    approved_cohort: 2026-09-C1
+    model_id: centml/DeepSeek-R1-NVFP4-v2-mlpinf
+    weight_checksum: git-sha1:93947a0d7bd04f73ff98636f6b18ff5839e7aaf9
+
+  # gpt-oss-120b: a separately published EAGLE3 head, carried over from legacy
+  # submissions. This is the repository nvidia/gpt-oss-120b-Eagle3 redirects to; the
+  # head is unquantized (hf_quant_config.json quant_algo: null).
+  - benchmark: gpt-oss-120b
+    approved_cohort: 2026-09-C1
+    model_id: nvidia/gpt-oss-120b-Eagle3-long-context
+    weight_checksum: git-sha1:633caf45f31288cbb70ee237f7c939db707ecc94

tests/submission_checker/test_checks_specdec_agentic.py

@@ -70,8 +70,10 @@ def _errors(binding: DrafterBinding, rule: str) -> list:
 @pytest.mark.unit
 class TestDrafterList:
     #: The approved heads in mlcommons/endpoints examples/10_Agentic_Inference/README.md,
-    #: "Approved Checkpoints and Speculative-Decoding Heads": benchmark -> {model_id: revision}.
-    #: Qwen and DeepSeek-V4.1-Flash heads are native to their approved checkpoints.
+    #: "Approved Checkpoints and Speculative-Decoding Heads", plus the DeepSeek-R1 and
+    #: GPT-OSS-120B heads carried over from legacy submissions: benchmark -> {model_id:
+    #: revision}. Qwen, DeepSeek-V4.1-Flash and DeepSeek-R1 heads are native to their
+    #: approved checkpoints.
     _REFERENCE_HEADS = {
         "kimi-k3": {
             "RadixArk/Kimi-K3-DSpark": "3c5bac301d9cf392706189d82ed947feca6c2f0f",
@@ -85,6 +87,13 @@ class TestDrafterList:
             "Qwen/Qwen3.6-35B-A3B-FP8": "95a723d08a9490559dae23d0cff1d9466213d989",
             "nvidia/Qwen3.6-35B-A3B-NVFP4": "1355db6a052410cfd62085d94b58866fd0f2c3c5",
         },
+        "deepseek-r1": {
+            "deepseek-ai/DeepSeek-R1": "56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad",
+            "centml/DeepSeek-R1-NVFP4-v2-mlpinf": "93947a0d7bd04f73ff98636f6b18ff5839e7aaf9",
+        },
+        "gpt-oss-120b": {
+            "nvidia/gpt-oss-120b-Eagle3-long-context": "633caf45f31288cbb70ee237f7c939db707ecc94",
+        },
     }
 
     def test_bundled_list_carries_the_reference_approved_heads(self) -> None:
@@ -99,10 +108,12 @@ def test_bundled_list_carries_the_reference_approved_heads(self) -> None:
         }
 
     def test_bundled_heads_record_the_2026_09_c1_cohort(self) -> None:
-        """The README published every head before the 2026-10-C0 publication.
+        """Every head shares the initial 2026-09-C1 approval cohort.
 
-        Qwen and Kimi on 2026-09-10 (endpoints#494), DeepSeek-V4.1-Flash on 2026-09-30
-        (endpoints#519): no earlier than each entry's own publication cohort.
+        Qwen and Kimi were published on 2026-09-10 (endpoints#494) and DeepSeek-V4.1-Flash on
+        2026-09-30 (endpoints#519), before the 2026-10-C0 publication. The DeepSeek-R1 and
+        GPT-OSS-120B heads are carried over from legacy submissions and recorded with them, so
+        that 2026-10-C1 submissions can use them.
         """
         cohorts = {
             d.model_id: d.approved_cohort for ds in load_approved_drafters().values() for d in ds