Send point.yaml and system_power.json, with derived power, on runs create

docs/endpoints-cli/usage/runs.md

@@ -70,6 +70,7 @@ A run flagged `--test` shows as `Test Run │ Yes` in `runs get`. It is **not**
 ├── system_desc.json                  # §8.2 hardware/software description (required)
 ├── point.yaml                        # §8.3 Pareto-point disclosure (required)
 │                                     #   ^ both authored by the submitter, not by endpoints
+├── system_power.json                 # §4.5.2 power descriptor (Standardized; submitter-authored)
 ├── performance/result_summary.json   # Performance metrics (required)
 ├── accuracy/accuracy_results.json    # Per-dataset accuracy scores
 ├── config.yaml                       # Resolved benchmark configuration (optional as of v1.0)
@@ -81,6 +82,14 @@ A run flagged `--test` shows as `Test Run │ Yes` in `runs get`. It is **not**
 └── documentation/                    # Merged into the bundle's shared docs/
 ```
 
+**What reaches the database.** Besides `system_desc.json`, `config.yaml` and the
+result summary, `runs create` sends `point.yaml` as `point_config`. When the folder has
+a `system_power.json`, it sends that too, as `system_power`, plus a `derived` object
+computed with the submission checker's own power arithmetic: `provisioned_power_kw`,
+`point_power_kw` (scaled by the point's `nodes_used`), `system_tps_per_kw`, and
+`mlc_estimated_power` (true when any value fell back to an Appendix D default).
+`--dry-run` prints the exact payload.
+
 This is the layout `mlcommons/endpoints` writes to its `report_dir` — `performance/`
 exists when the performance phase ran, `accuracy/` when the accuracy phase ran, and
 `--mode both` produces both. See

src/endpoints_submission_cli/runs/parser.py

@@ -10,6 +10,9 @@
         point.yaml                          — §8.3 Pareto-point disclosure        [required]
                                               Neither is an endpoints artifact: the
                                               submitter authors both and drops them in
+        system_power.json                   — §4.5.2 provisioned-power descriptor [optional]
+                                              also submitter-authored; required for
+                                              Standardized at submission time
         performance/result_summary.json     — performance metrics                 [required]
         config.yaml                         — resolved benchmark configuration    [optional]
         accuracy/accuracy_results.json      — per-dataset accuracy scores
@@ -35,6 +38,9 @@
 
 import yaml
 
+from submission_checker.layout import SYSTEM_POWER_JSON
+from submission_checker.power_summary import derive_point_power
+
 from ..exceptions import RunFolderError
 from ..truncation import truncate_responses
 
@@ -73,7 +79,10 @@ def parse_run_folder(path: Path) -> dict[str, Any]:
               passed through as an empty mapping when absent.
 
     Returns:
-        Dict with keys matching ``RunCreate`` schema fields.
+        Dict with keys matching ``RunCreate`` schema fields. ``point_config`` is
+        ``point.yaml``. ``system_power`` is ``system_power.json`` with a ``derived``
+        object added (see :func:`derive_point_power`), or absent when the folder has
+        no descriptor.
 
     Raises:
         RunFolderError: If any required file is absent or malformed.
@@ -88,18 +97,30 @@ def parse_run_folder(path: Path) -> dict[str, Any]:
     config_path = path / "config.yaml"
     config = _load_yaml(config_path) if config_path.exists() else {}
     result_summary = _load_json(path / _RESULT_SUMMARY_REL)
+    point_config = _load_yaml(path / "point.yaml")
 
     started_at, finished_at = _extract_timestamps(result_summary)
     benchmark_version = result_summary.get("git_sha") or "unknown"
 
-    return {
+    payload: dict[str, Any] = {
         "benchmark_version": benchmark_version,
         "started_at": started_at.isoformat(),
         "finished_at": finished_at.isoformat(),
         "system_info": system_info,
         "config": config,
         "result_summary": result_summary,
+        "point_config": point_config,
     }
+    power_path = path / SYSTEM_POWER_JSON
+    if power_path.exists():
+        system_power = _load_json(power_path)
+        # Computed here so the database and visualizer get the same figures the
+        # checker accepts; the file in the archive stays as the submitter wrote it.
+        payload["system_power"] = {
+            **system_power,
+            "derived": derive_point_power(system_power, point_config, system_info, result_summary),
+        }
+    return payload
 
 
 def _validate_required_files(path: Path) -> None:

src/submission_checker/power_summary.py

@@ -0,0 +1,148 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""One point's power figures, for storing beside the run rather than checking it.
+
+The checker computes a point's provisioned and per-point power to *verify* what the
+submitter wrote. Consumers of the results — the database, the visualizer — want the
+same figures as data: provisioned power, the power the point engages, throughput per
+kilowatt, and whether any value fell back to an Appendix D default ("MLC Estimated
+Power"). :func:`derive_point_power` produces them with the checker's own arithmetic
+(:meth:`SystemPower.compute` and :meth:`PowerComputation.point_power_kw`), so a stored
+figure can never disagree with what compliance accepted.
+
+It works on parsed file contents, not paths, because its callers hold them already:
+``runs create`` from the run folder, and the review-repo sync job from a PR's files.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import ValidationError
+
+from .models.file.system_power import PowerComputation, SystemPower
+
+__all__ = ["derive_point_power", "host_cores_by_ensemble"]
+
+_NS_PER_S = 1e9
+
+
+def host_cores_by_ensemble(system_desc: dict[str, Any]) -> dict[int, int]:
+    """``host_processor_core_count`` per ``system_node_ensemble_id``, for D.2's default.
+
+    Read from §8.2.1's ``node_types``; an ensemble id given as a digit string is
+    coerced, as :class:`~submission_checker.models.NodeType` does.
+    """
+    cores: dict[int, int] = {}
+    for node in system_desc.get("node_types") or []:
+        if not isinstance(node, dict):
+            continue
+        ensemble = node.get("system_node_ensemble_id")
+        if isinstance(ensemble, str) and ensemble.strip().isdigit():
+            ensemble = int(ensemble)
+        count = node.get("host_processor_core_count")
+        if isinstance(ensemble, int) and isinstance(count, int):
+            cores[ensemble] = count
+    return cores
+
+
+def _system_tps(result_summary: dict[str, Any]) -> float | None:
+    """``total output tokens / elapsed seconds``, as ``PointSummary.system_tps``."""
+    lengths = result_summary.get("output_sequence_lengths")
+    duration = _number(result_summary.get("duration_ns"))
+    total = _number(lengths.get("total")) if isinstance(lengths, dict) else None
+    if duration is None or total is None or duration <= 0:
+        return None
+    return total / (duration / _NS_PER_S)
+
+
+def _number(value: object) -> float | None:
+    """A JSON number as float; ``None`` for bools and anything non-numeric."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value)
+
+
+def _nodes_by_set(point_config: dict[str, Any], power: PowerComputation) -> dict[int, int] | None:
+    """The point's ``nodes_used`` keyed by ``node_set_id``, or ``None`` if unusable.
+
+    An absent declaration is ``{}`` — every set fully engaged, §4.5.3's default. One
+    that names an ensemble no single set matches, or more nodes than are provisioned,
+    is the error the checker's ``nodes-used`` rule reports; no figure is derived from
+    it rather than guess.
+    """
+    declared = point_config.get("nodes_used")
+    if declared is None:
+        return {}
+    if not isinstance(declared, list):
+        return None
+    by_set: dict[int, int] = {}
+    for entry in declared:
+        if not isinstance(entry, dict):
+            return None
+        ensemble, nodes = entry.get("system_node_ensemble_id"), entry.get("nodes")
+        matches = [s for s in power.sets if s.system_node_ensemble_id == ensemble]
+        if len(matches) != 1 or not isinstance(nodes, int):
+            return None
+        node_set = matches[0]
+        if not 1 <= nodes <= node_set.nodes_provisioned or node_set.node_set_id in by_set:
+            return None
+        by_set[node_set.node_set_id] = nodes
+    return by_set
+
+
+def derive_point_power(
+    system_power: dict[str, Any],
+    point_config: dict[str, Any],
+    system_desc: dict[str, Any],
+    result_summary: dict[str, Any],
+) -> dict[str, Any]:
+    """Power figures for one point, as a JSON-ready mapping.
+
+    Args:
+        system_power: The system's ``system_power.json``.
+        point_config: The point's ``point.yaml``.
+        system_desc: The point's ``system_desc.json`` (for D.2's core counts).
+        result_summary: The point's ``performance/result_summary.json``.
+
+    Returns:
+        ``provisioned_power_kw``, ``point_power_kw`` and ``system_tps_per_kw`` (each
+        ``None`` where it cannot be determined), ``mlc_estimated_power``, and the
+        ``estimated`` values and ``problems`` behind them. Never raises on bad input:
+        a descriptor the checker would reject yields ``None`` figures and says why.
+    """
+    out: dict[str, Any] = {
+        "provisioned_power_kw": None,
+        "point_power_kw": None,
+        "system_tps_per_kw": None,
+        "mlc_estimated_power": False,
+        "estimated": [],
+        "problems": [],
+    }
+    try:
+        model = SystemPower.model_validate(system_power)
+    except ValidationError as exc:
+        out["problems"] = [f"system_power.json is invalid: {exc.error_count()} error(s)"]
+        return out
+
+    power = model.compute(host_cores_by_ensemble(system_desc))
+    out["provisioned_power_kw"] = power.provisioned_power_kw
+    out["estimated"] = list(power.estimated)
+    out["problems"] = list(power.problems)
+    # The submitter may also declare the tag outright (Appendix E's own field).
+    out["mlc_estimated_power"] = bool(power.estimated) or model.mlc_estimated_power is True
+    if power.provisioned_power_kw is None:
+        return out
+
+    nodes = _nodes_by_set(point_config, power)
+    if nodes is None:
+        out["problems"].append("nodes_used does not match the power descriptor's node sets")
+        return out
+    point_kw = power.point_power_kw(nodes)
+    out["point_power_kw"] = point_kw
+
+    tps = _system_tps(result_summary)
+    if point_kw and tps is not None:
+        # §4.5.2 reports it to one decimal place.
+        out["system_tps_per_kw"] = round(tps / point_kw, 1)
+    return out

tests/endpoints_submission_cli/runs/test_parser.py

@@ -41,6 +41,33 @@ def test_valid_folder_returns_payload(self, endpoints_run_folder: Path) -> None:
         assert "config" in payload
         assert "result_summary" in payload
 
+    def test_point_yaml_sent_as_point_config(self, endpoints_run_folder: Path) -> None:
+        payload = parse_run_folder(endpoints_run_folder)
+        point = yaml.safe_load((endpoints_run_folder / "point.yaml").read_text())
+        assert payload["point_config"] == point
+
+    def test_system_power_sent_with_derived_figures(self, endpoints_run_folder: Path) -> None:
+        payload = parse_run_folder(endpoints_run_folder)
+        raw = json.loads((endpoints_run_folder / "system_power.json").read_text())
+        power = payload["system_power"]
+        # The descriptor itself passes through untouched.
+        assert {k: v for k, v in power.items() if k != "derived"} == raw
+        derived = power["derived"]
+        # 2 x 350 W CPU + 8 x 700 W GPU + 3500 W scale-up, +50% air overhead.
+        assert derived["provisioned_power_kw"] == 14.7
+        assert derived["point_power_kw"] == 14.7  # one node, fully engaged
+        summary = payload["result_summary"]
+        tps = summary["output_sequence_lengths"]["total"] / (summary["duration_ns"] / 1e9)
+        assert derived["system_tps_per_kw"] == round(tps / 14.7, 1)
+        assert derived["mlc_estimated_power"] is False
+        assert derived["problems"] == []
+
+    def test_no_system_power_file_omits_the_field(self, endpoints_run_folder: Path) -> None:
+        (endpoints_run_folder / "system_power.json").unlink()
+        payload = parse_run_folder(endpoints_run_folder)
+        assert "system_power" not in payload
+        assert "point_config" in payload
+
     def test_system_info_passthrough(self, endpoints_run_folder: Path) -> None:
         payload = parse_run_folder(endpoints_run_folder)
         si = payload["system_info"]

tests/submission_checker/test_power_summary.py

@@ -0,0 +1,121 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons
+# SPDX-License-Identifier: Apache-2.0
+"""Unit tests for submission_checker.power_summary."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any
+
+import pytest
+
+from submission_checker.power_summary import derive_point_power, host_cores_by_ensemble
+
+_SOURCE = {"source_type": "vendor_spec", "source": "https://example.com/x"}
+
+
+def _power(nodes: int = 2, **extra: Any) -> dict[str, Any]:
+    """A two-node air-cooled descriptor: 9.8 kW major per node, 14.7 kW with overhead."""
+    return {
+        "system_desc_id": "sys",
+        "cooling": "air",
+        "node_sets": [
+            {
+                "node_set_id": 0,
+                "system_node_ensemble_id": 0,
+                "nodes_provisioned": nodes,
+                "power_method": "component_sum",
+                "components": {
+                    "cpu": {"count_per_node": 2, "tdp_per_unit": {"value_w": 350, **_SOURCE}},
+                    "accelerator": {
+                        "count_per_node": 8,
+                        "tdp_per_unit": {"value_w": 700, **_SOURCE},
+                    },
+                    "scale_up_network": {
+                        "method": "declared_tdp",
+                        "switch_count": 1,
+                        "tdp_per_switch": {"value_w": 3500, **_SOURCE},
+                    },
+                },
+            }
+        ],
+        "scale_out": {"present": False},
+        **extra,
+    }
+
+
+#: 29.4 kW provisioned / 14.7 kW for one of two nodes; 2940 tokens/s.
+_SUMMARY = {"duration_ns": 10 * 10**9, "output_sequence_lengths": {"total": 29_400}}
+
+
+@pytest.mark.unit
+class TestDerivePointPower:
+    def test_fully_engaged_point(self) -> None:
+        out = derive_point_power(_power(), {}, {}, _SUMMARY)
+        assert out["provisioned_power_kw"] == 29.4
+        assert out["point_power_kw"] == 29.4
+        assert out["system_tps_per_kw"] == 100.0
+        assert out["mlc_estimated_power"] is False
+        assert out["problems"] == []
+
+    def test_point_scaled_by_nodes_used(self) -> None:
+        point = {"nodes_used": [{"system_node_ensemble_id": 0, "nodes": 1}]}
+        out = derive_point_power(_power(), point, {}, _SUMMARY)
+        assert out["provisioned_power_kw"] == 29.4
+        assert out["point_power_kw"] == 14.7
+        assert out["system_tps_per_kw"] == 200.0
+
+    def test_appendix_d_default_sets_the_estimated_flag(self) -> None:
+        power = _power()
+        cpu = power["node_sets"][0]["components"]["cpu"]
+        cpu["tdp_per_unit"] = {"value_w": 350, "source_type": "mlc_default", "source": "D.2"}
+        out = derive_point_power(power, {}, {}, _SUMMARY)
+        assert out["mlc_estimated_power"] is True
+        assert out["estimated"]
+
+    def test_submitter_declared_estimate_is_kept(self) -> None:
+        out = derive_point_power(_power(mlc_estimated_power=True), {}, {}, _SUMMARY)
+        assert out["mlc_estimated_power"] is True
+
+    def test_nodes_used_beyond_provisioned_derives_no_point_figure(self) -> None:
+        point = {"nodes_used": [{"system_node_ensemble_id": 0, "nodes": 3}]}
+        out = derive_point_power(_power(), point, {}, _SUMMARY)
+        assert out["provisioned_power_kw"] == 29.4
+        assert out["point_power_kw"] is None
+        assert out["system_tps_per_kw"] is None
+        assert out["problems"]
+
+    def test_unknown_ensemble_derives_no_point_figure(self) -> None:
+        point = {"nodes_used": [{"system_node_ensemble_id": 7, "nodes": 1}]}
+        out = derive_point_power(_power(), point, {}, _SUMMARY)
+        assert out["point_power_kw"] is None
+
+    def test_invalid_descriptor_never_raises(self) -> None:
+        out = derive_point_power({"cooling": "steam"}, {}, {}, _SUMMARY)
+        assert out["provisioned_power_kw"] is None
+        assert out["problems"]
+
+    def test_missing_throughput_leaves_tps_per_kw_unset(self) -> None:
+        out = derive_point_power(_power(), {}, {}, {})
+        assert out["point_power_kw"] == 29.4
+        assert out["system_tps_per_kw"] is None
+
+    def test_inputs_not_mutated(self) -> None:
+        power = _power()
+        before = copy.deepcopy(power)
+        derive_point_power(power, {}, {}, _SUMMARY)
+        assert power == before
+
+
+@pytest.mark.unit
+class TestHostCoresByEnsemble:
+    def test_reads_node_types_and_coerces_ids(self) -> None:
+        desc = {
+            "node_types": [
+                {"system_node_ensemble_id": "0", "host_processor_core_count": 64},
+                {"system_node_ensemble_id": 1, "host_processor_core_count": 32},
+                {"system_node_ensemble_id": 2},
+                "junk",
+            ]
+        }
+        assert host_cores_by_ensemble(desc) == {0: 64, 1: 32}