Stop treating pipeline-assigned fields as system_desc.json fields

src/endpoints_submission_cli/submissions/builder.py

@@ -40,6 +40,7 @@
 
 from submission_checker import layout
 from submission_checker.models.file import SystemDescription
+from submission_checker.models.file.system import PIPELINE_ASSIGNED_FIELDS
 
 from ..exceptions import SubmissionBuildError
 from ..truncation import truncate_responses
@@ -315,6 +316,11 @@ def _load_system_desc(base: Path, run_id: str, division: str, availability: str)
     for legacy in ("system_availability_status", "availability_status"):
         raw.pop(legacy, None)
     raw["publication_status"] = availability
+    # The pipeline assigns the submission ID and dates (§8.2), so a value in the
+    # submitter's file — usually left over from a v0.7 template — is stale. Drop it
+    # rather than publish it.
+    for assigned in PIPELINE_ASSIGNED_FIELDS:
+        raw.pop(assigned, None)
     try:
         sd = SystemDescription.model_validate(raw)
     except ValidationError as exc:

src/submission_checker/checker.py

@@ -42,6 +42,7 @@
 from .models import err as _err
 from .models import ok as _ok
 from .models import warn as _warn
+from .models.file.system import PIPELINE_ASSIGNED_FIELDS
 from .models.file.system_power import AIR_COOLED_OVERHEAD, overhead_for_cooling
 from .models.loader import (
     load_accuracy_result,
@@ -128,6 +129,12 @@ def _results_has_accuracy_scores(path: Path) -> bool:
     }
 )
 
+#: Keys of ``system_desc.json`` that are not §8.2 fields and are never compared.
+#: The pipeline assigns the submission ID and dates (§8.2); a copy left in a
+#: submitter's file from a v0.7 template says nothing about the system, so differing
+#: copies are not flagged.
+_IGNORED_SYSTEM_DESC_FIELDS = frozenset(PIPELINE_ASSIGNED_FIELDS)
+
 #: §4.5 scope: "Power normalization applies to all Standardized division submissions
 #: … RDI submissions MAY report normalized throughput but are not required to", and
 #: Serviced normalisation "will be introduced in a later version".
@@ -185,7 +192,11 @@ class _SystemFacts:
 
 def _system_desc_identity(data: dict[str, object]) -> dict[str, object]:
     """Strip the fields that legitimately vary between points of the same curve."""
-    return {k: v for k, v in data.items() if k not in _PER_POINT_SYSTEM_DESC_FIELDS}
+    return {
+        k: v
+        for k, v in data.items()
+        if k not in _PER_POINT_SYSTEM_DESC_FIELDS and k not in _IGNORED_SYSTEM_DESC_FIELDS
+    }
 
 
 def _read_json(path: Path) -> dict[str, object] | None:

src/submission_checker/models/file/system.py

@@ -23,6 +23,7 @@
     "DatasetAccuracyScores",
     "Division",
     "NodeType",
+    "PIPELINE_ASSIGNED_FIELDS",
     "SystemAvailabilityStatus",
     "SystemDescription",
 ]
@@ -54,6 +55,18 @@ class SystemAvailabilityStatus(str, Enum):
     "accelerator_host_interconnect",
 )
 
+#: Keys v0.7's system description carried that the submission pipeline now assigns,
+#: so they are not §8.2 fields. The submission ID (§8.5) is the ``<submission_id>/``
+#: directory (§8.1); the submission and publication dates are set when the bundle is
+#: submitted and published.
+#:
+#: :class:`SystemDescription` does not declare them. Declaring them made
+#: ``model_dump`` write each one as ``null`` into every bundled system_desc.json, and
+#: typed them as strings, so a v0.7 numeric id failed validation. Left undeclared,
+#: ``extra="allow"`` still parses a leftover key of any type; the builder drops them
+#: and the consistency check ignores them.
+PIPELINE_ASSIGNED_FIELDS = ("submission_id", "submission_date", "publish_date")
+
 
 class AcceleratorInfo(BaseModel):
     """One accelerator configuration within a node type (§8.2.1).
@@ -190,13 +203,12 @@ class SystemDescription(BaseModel):
 
     model_config = ConfigDict(extra="allow")
 
-    # Org / submission metadata. §8.2 moved these out of the system description in
-    # v1.0; still read when present because the existing corpus carries them.
+    # Org metadata. §8.2 moved these out of the system description in v1.0; still read
+    # when present because the existing corpus carries them.
+    #
+    # :data:`PIPELINE_ASSIGNED_FIELDS` are deliberately not declared: see there.
     submitter_org_names: str | None = None
     submitter_contact: str | None = None
-    submission_id: str | None = None
-    submission_date: str | None = None
-    publish_date: str | None = None
 
     # System metadata
     system_name: str

tests/endpoints_submission_cli/submissions/test_builder.py

@@ -115,6 +115,31 @@ def test_system_desc_written_per_point(self, run_archive: Path, tmp_path: Path)
         assert data["division"] == "Standardized"
         assert "node_types" in data
 
+    @pytest.mark.parametrize("field", ["submission_id", "submission_date", "publish_date"])
+    @pytest.mark.parametrize("leftover", [None, "v07-leftover", 1234])
+    def test_system_desc_carries_no_pipeline_assigned_field(
+        self, run_folder: Path, tmp_path: Path, field: str, leftover: object
+    ) -> None:
+        """§8.2: the pipeline assigns these, so they are never system_desc.json keys.
+
+        ``None`` means the run's file has no key at all: the builder must not add one.
+        Otherwise a leftover from a v0.7 template is dropped rather than published.
+        """
+        if leftover is not None:
+            sd_path = run_folder / "system_desc.json"
+            sd = json.loads(sd_path.read_text())
+            sd_path.write_text(json.dumps({**sd, field: leftover}))
+        archive = tmp_path / "run.tar.gz"
+        with tarfile.open(archive, "w:gz") as tar:
+            tar.add(run_folder, arcname=run_folder.name)
+        sub_dir = _submission_root(
+            build_submission_folder(
+                [("run-001", archive)], "standardized", "available", tmp_path / "out", "sub-123"
+            )
+        )
+        (written,) = sub_dir.rglob("system_desc.json")
+        assert field not in json.loads(written.read_text())
+
     def test_point_yaml_created(self, run_archive: Path, tmp_path: Path) -> None:
         sub_dir = build_submission_folder(
             [("run-001", run_archive)], "standardized", "available", tmp_path

tests/submission_checker/test_checker.py

@@ -420,6 +420,21 @@ def test_tps_utilization_may_differ_across_a_curve(self, tmp_path):
         report = _check(root)
         assert not _errors(report, "system-description-consistency")
 
+    @pytest.mark.parametrize("field", ["submission_id", "submission_date", "publish_date"])
+    @pytest.mark.parametrize("leftover", ["v07-leftover", 1234])
+    def test_leftover_pipeline_assigned_field_is_not_flagged(self, tmp_path, field, leftover):
+        """§8.2: the pipeline assigns these, so a stale copy is ignored.
+
+        Covers both ways it used to fail: a v0.7 numeric value was a schema error, and
+        points carrying different values failed the consistency check.
+        """
+        root = _build_submission(tmp_path, system_desc={**_SYSTEM_DESC, field: leftover})
+        changed = root / "results" / "test-sys" / "llama3-70b" / "r88" / "system_desc.json"
+        changed.write_text(json.dumps({**_SYSTEM_DESC, field: "another-value"}))
+        report = _check(root)
+        assert not _errors(report, "system-description-consistency")
+        assert not _errors(report, "system-description-valid")
+
     def test_empty_system_results_dir(self, tmp_path):
         """benchmark-model-dir error when results/<system>/ has no subdirectories."""
         (tmp_path / "results" / "test-sys").mkdir(parents=True)

tests/submission_checker/test_models.py

@@ -198,6 +198,23 @@ def test_fields_moved_to_point_yaml_are_optional(field):
     assert getattr(sd, field) is None
 
 
+#: v0.7 system-description keys the submission pipeline now assigns (§8.2).
+_PIPELINE_ASSIGNED = ("submission_id", "submission_date", "publish_date")
+
+
+@pytest.mark.parametrize("field", _PIPELINE_ASSIGNED)
+def test_pipeline_assigned_field_is_not_added_to_the_system_description(field):
+    """§8.2: the pipeline assigns these; a dump must not invent them."""
+    assert field not in SystemDescription(**_BASE_FLAT).model_dump(mode="json")
+
+
+@pytest.mark.parametrize("field", _PIPELINE_ASSIGNED)
+@pytest.mark.parametrize("leftover", ["v07-leftover", 1234, None])
+def test_leftover_pipeline_assigned_field_of_any_type_parses(field, leftover):
+    """A v0.7 template's copy is not a §8.2 field, so its type is not checked."""
+    SystemDescription(**{**_BASE_FLAT, field: leftover})
+
+
 @pytest.mark.parametrize("field", ["model_id", "model_name"])
 def test_fields_the_rules_dropped_are_no_longer_declared(field):
     """Policies PR #130 removed these from §8.2 entirely, so nothing reads them.