From 369c775fd40739d281751949795f8ed5b64dacd1 Mon Sep 17 00:00:00 2001 From: Arav Agarwal Date: Sat, 10 Oct 2026 22:29:47 -0400 Subject: [PATCH] Send point.yaml and system_power.json with derived power on runs create runs create now sends point.yaml as point_config and, when present, system_power.json as system_power with a derived object computed by the checker's own power code: provisioned_power_kw, point_power_kw, system_tps_per_kw and mlc_estimated_power. The new submission_checker.power_summary.derive_point_power is importable so the review-repo sync job computes the same figures. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/endpoints-cli/usage/runs.md | 9 ++ src/endpoints_submission_cli/runs/parser.py | 25 ++- src/submission_checker/power_summary.py | 148 ++++++++++++++++++ .../runs/test_parser.py | 27 ++++ .../submission_checker/test_power_summary.py | 121 ++++++++++++++ 5 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 src/submission_checker/power_summary.py create mode 100644 tests/submission_checker/test_power_summary.py diff --git a/docs/endpoints-cli/usage/runs.md b/docs/endpoints-cli/usage/runs.md index da8e9135..51da6029 100644 --- a/docs/endpoints-cli/usage/runs.md +++ b/docs/endpoints-cli/usage/runs.md @@ -70,6 +70,7 @@ A run flagged `--test` shows as `Test Run │ Yes` in `runs get`. It is **not** ├── system_desc.json # §8.2 hardware/software description (required) ├── point.yaml # §8.3 Pareto-point disclosure (required) │ # ^ both authored by the submitter, not by endpoints +├── system_power.json # §4.5.2 power descriptor (Standardized; submitter-authored) ├── performance/result_summary.json # Performance metrics (required) ├── accuracy/accuracy_results.json # Per-dataset accuracy scores ├── config.yaml # Resolved benchmark configuration (optional as of v1.0) @@ -81,6 +82,14 @@ A run flagged `--test` shows as `Test Run │ Yes` in `runs get`. It is **not** └── documentation/ # Merged into the bundle's shared docs/ ``` +**What reaches the database.** Besides `system_desc.json`, `config.yaml` and the +result summary, `runs create` sends `point.yaml` as `point_config`. When the folder has +a `system_power.json`, it sends that too, as `system_power`, plus a `derived` object +computed with the submission checker's own power arithmetic: `provisioned_power_kw`, +`point_power_kw` (scaled by the point's `nodes_used`), `system_tps_per_kw`, and +`mlc_estimated_power` (true when any value fell back to an Appendix D default). +`--dry-run` prints the exact payload. + This is the layout `mlcommons/endpoints` writes to its `report_dir` — `performance/` exists when the performance phase ran, `accuracy/` when the accuracy phase ran, and `--mode both` produces both. See diff --git a/src/endpoints_submission_cli/runs/parser.py b/src/endpoints_submission_cli/runs/parser.py index ed1cb521..69b3d2a1 100644 --- a/src/endpoints_submission_cli/runs/parser.py +++ b/src/endpoints_submission_cli/runs/parser.py @@ -10,6 +10,9 @@ point.yaml — §8.3 Pareto-point disclosure [required] Neither is an endpoints artifact: the submitter authors both and drops them in + system_power.json — §4.5.2 provisioned-power descriptor [optional] + also submitter-authored; required for + Standardized at submission time performance/result_summary.json — performance metrics [required] config.yaml — resolved benchmark configuration [optional] accuracy/accuracy_results.json — per-dataset accuracy scores @@ -35,6 +38,9 @@ import yaml +from submission_checker.layout import SYSTEM_POWER_JSON +from submission_checker.power_summary import derive_point_power + from ..exceptions import RunFolderError from ..truncation import truncate_responses @@ -73,7 +79,10 @@ def parse_run_folder(path: Path) -> dict[str, Any]: passed through as an empty mapping when absent. Returns: - Dict with keys matching ``RunCreate`` schema fields. + Dict with keys matching ``RunCreate`` schema fields. ``point_config`` is + ``point.yaml``. ``system_power`` is ``system_power.json`` with a ``derived`` + object added (see :func:`derive_point_power`), or absent when the folder has + no descriptor. Raises: RunFolderError: If any required file is absent or malformed. @@ -88,18 +97,30 @@ def parse_run_folder(path: Path) -> dict[str, Any]: config_path = path / "config.yaml" config = _load_yaml(config_path) if config_path.exists() else {} result_summary = _load_json(path / _RESULT_SUMMARY_REL) + point_config = _load_yaml(path / "point.yaml") started_at, finished_at = _extract_timestamps(result_summary) benchmark_version = result_summary.get("git_sha") or "unknown" - return { + payload: dict[str, Any] = { "benchmark_version": benchmark_version, "started_at": started_at.isoformat(), "finished_at": finished_at.isoformat(), "system_info": system_info, "config": config, "result_summary": result_summary, + "point_config": point_config, } + power_path = path / SYSTEM_POWER_JSON + if power_path.exists(): + system_power = _load_json(power_path) + # Computed here so the database and visualizer get the same figures the + # checker accepts; the file in the archive stays as the submitter wrote it. + payload["system_power"] = { + **system_power, + "derived": derive_point_power(system_power, point_config, system_info, result_summary), + } + return payload def _validate_required_files(path: Path) -> None: diff --git a/src/submission_checker/power_summary.py b/src/submission_checker/power_summary.py new file mode 100644 index 00000000..53ce5ed2 --- /dev/null +++ b/src/submission_checker/power_summary.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""One point's power figures, for storing beside the run rather than checking it. + +The checker computes a point's provisioned and per-point power to *verify* what the +submitter wrote. Consumers of the results — the database, the visualizer — want the +same figures as data: provisioned power, the power the point engages, throughput per +kilowatt, and whether any value fell back to an Appendix D default ("MLC Estimated +Power"). :func:`derive_point_power` produces them with the checker's own arithmetic +(:meth:`SystemPower.compute` and :meth:`PowerComputation.point_power_kw`), so a stored +figure can never disagree with what compliance accepted. + +It works on parsed file contents, not paths, because its callers hold them already: +``runs create`` from the run folder, and the review-repo sync job from a PR's files. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import ValidationError + +from .models.file.system_power import PowerComputation, SystemPower + +__all__ = ["derive_point_power", "host_cores_by_ensemble"] + +_NS_PER_S = 1e9 + + +def host_cores_by_ensemble(system_desc: dict[str, Any]) -> dict[int, int]: + """``host_processor_core_count`` per ``system_node_ensemble_id``, for D.2's default. + + Read from §8.2.1's ``node_types``; an ensemble id given as a digit string is + coerced, as :class:`~submission_checker.models.NodeType` does. + """ + cores: dict[int, int] = {} + for node in system_desc.get("node_types") or []: + if not isinstance(node, dict): + continue + ensemble = node.get("system_node_ensemble_id") + if isinstance(ensemble, str) and ensemble.strip().isdigit(): + ensemble = int(ensemble) + count = node.get("host_processor_core_count") + if isinstance(ensemble, int) and isinstance(count, int): + cores[ensemble] = count + return cores + + +def _system_tps(result_summary: dict[str, Any]) -> float | None: + """``total output tokens / elapsed seconds``, as ``PointSummary.system_tps``.""" + lengths = result_summary.get("output_sequence_lengths") + duration = _number(result_summary.get("duration_ns")) + total = _number(lengths.get("total")) if isinstance(lengths, dict) else None + if duration is None or total is None or duration <= 0: + return None + return total / (duration / _NS_PER_S) + + +def _number(value: object) -> float | None: + """A JSON number as float; ``None`` for bools and anything non-numeric.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + + +def _nodes_by_set(point_config: dict[str, Any], power: PowerComputation) -> dict[int, int] | None: + """The point's ``nodes_used`` keyed by ``node_set_id``, or ``None`` if unusable. + + An absent declaration is ``{}`` — every set fully engaged, §4.5.3's default. One + that names an ensemble no single set matches, or more nodes than are provisioned, + is the error the checker's ``nodes-used`` rule reports; no figure is derived from + it rather than guess. + """ + declared = point_config.get("nodes_used") + if declared is None: + return {} + if not isinstance(declared, list): + return None + by_set: dict[int, int] = {} + for entry in declared: + if not isinstance(entry, dict): + return None + ensemble, nodes = entry.get("system_node_ensemble_id"), entry.get("nodes") + matches = [s for s in power.sets if s.system_node_ensemble_id == ensemble] + if len(matches) != 1 or not isinstance(nodes, int): + return None + node_set = matches[0] + if not 1 <= nodes <= node_set.nodes_provisioned or node_set.node_set_id in by_set: + return None + by_set[node_set.node_set_id] = nodes + return by_set + + +def derive_point_power( + system_power: dict[str, Any], + point_config: dict[str, Any], + system_desc: dict[str, Any], + result_summary: dict[str, Any], +) -> dict[str, Any]: + """Power figures for one point, as a JSON-ready mapping. + + Args: + system_power: The system's ``system_power.json``. + point_config: The point's ``point.yaml``. + system_desc: The point's ``system_desc.json`` (for D.2's core counts). + result_summary: The point's ``performance/result_summary.json``. + + Returns: + ``provisioned_power_kw``, ``point_power_kw`` and ``system_tps_per_kw`` (each + ``None`` where it cannot be determined), ``mlc_estimated_power``, and the + ``estimated`` values and ``problems`` behind them. Never raises on bad input: + a descriptor the checker would reject yields ``None`` figures and says why. + """ + out: dict[str, Any] = { + "provisioned_power_kw": None, + "point_power_kw": None, + "system_tps_per_kw": None, + "mlc_estimated_power": False, + "estimated": [], + "problems": [], + } + try: + model = SystemPower.model_validate(system_power) + except ValidationError as exc: + out["problems"] = [f"system_power.json is invalid: {exc.error_count()} error(s)"] + return out + + power = model.compute(host_cores_by_ensemble(system_desc)) + out["provisioned_power_kw"] = power.provisioned_power_kw + out["estimated"] = list(power.estimated) + out["problems"] = list(power.problems) + # The submitter may also declare the tag outright (Appendix E's own field). + out["mlc_estimated_power"] = bool(power.estimated) or model.mlc_estimated_power is True + if power.provisioned_power_kw is None: + return out + + nodes = _nodes_by_set(point_config, power) + if nodes is None: + out["problems"].append("nodes_used does not match the power descriptor's node sets") + return out + point_kw = power.point_power_kw(nodes) + out["point_power_kw"] = point_kw + + tps = _system_tps(result_summary) + if point_kw and tps is not None: + # §4.5.2 reports it to one decimal place. + out["system_tps_per_kw"] = round(tps / point_kw, 1) + return out diff --git a/tests/endpoints_submission_cli/runs/test_parser.py b/tests/endpoints_submission_cli/runs/test_parser.py index 80485a6c..eadb4410 100644 --- a/tests/endpoints_submission_cli/runs/test_parser.py +++ b/tests/endpoints_submission_cli/runs/test_parser.py @@ -41,6 +41,33 @@ def test_valid_folder_returns_payload(self, endpoints_run_folder: Path) -> None: assert "config" in payload assert "result_summary" in payload + def test_point_yaml_sent_as_point_config(self, endpoints_run_folder: Path) -> None: + payload = parse_run_folder(endpoints_run_folder) + point = yaml.safe_load((endpoints_run_folder / "point.yaml").read_text()) + assert payload["point_config"] == point + + def test_system_power_sent_with_derived_figures(self, endpoints_run_folder: Path) -> None: + payload = parse_run_folder(endpoints_run_folder) + raw = json.loads((endpoints_run_folder / "system_power.json").read_text()) + power = payload["system_power"] + # The descriptor itself passes through untouched. + assert {k: v for k, v in power.items() if k != "derived"} == raw + derived = power["derived"] + # 2 x 350 W CPU + 8 x 700 W GPU + 3500 W scale-up, +50% air overhead. + assert derived["provisioned_power_kw"] == 14.7 + assert derived["point_power_kw"] == 14.7 # one node, fully engaged + summary = payload["result_summary"] + tps = summary["output_sequence_lengths"]["total"] / (summary["duration_ns"] / 1e9) + assert derived["system_tps_per_kw"] == round(tps / 14.7, 1) + assert derived["mlc_estimated_power"] is False + assert derived["problems"] == [] + + def test_no_system_power_file_omits_the_field(self, endpoints_run_folder: Path) -> None: + (endpoints_run_folder / "system_power.json").unlink() + payload = parse_run_folder(endpoints_run_folder) + assert "system_power" not in payload + assert "point_config" in payload + def test_system_info_passthrough(self, endpoints_run_folder: Path) -> None: payload = parse_run_folder(endpoints_run_folder) si = payload["system_info"] diff --git a/tests/submission_checker/test_power_summary.py b/tests/submission_checker/test_power_summary.py new file mode 100644 index 00000000..c5bbb0d5 --- /dev/null +++ b/tests/submission_checker/test_power_summary.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for submission_checker.power_summary.""" + +from __future__ import annotations + +import copy +from typing import Any + +import pytest + +from submission_checker.power_summary import derive_point_power, host_cores_by_ensemble + +_SOURCE = {"source_type": "vendor_spec", "source": "https://example.com/x"} + + +def _power(nodes: int = 2, **extra: Any) -> dict[str, Any]: + """A two-node air-cooled descriptor: 9.8 kW major per node, 14.7 kW with overhead.""" + return { + "system_desc_id": "sys", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": nodes, + "power_method": "component_sum", + "components": { + "cpu": {"count_per_node": 2, "tdp_per_unit": {"value_w": 350, **_SOURCE}}, + "accelerator": { + "count_per_node": 8, + "tdp_per_unit": {"value_w": 700, **_SOURCE}, + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": {"value_w": 3500, **_SOURCE}, + }, + }, + } + ], + "scale_out": {"present": False}, + **extra, + } + + +#: 29.4 kW provisioned / 14.7 kW for one of two nodes; 2940 tokens/s. +_SUMMARY = {"duration_ns": 10 * 10**9, "output_sequence_lengths": {"total": 29_400}} + + +@pytest.mark.unit +class TestDerivePointPower: + def test_fully_engaged_point(self) -> None: + out = derive_point_power(_power(), {}, {}, _SUMMARY) + assert out["provisioned_power_kw"] == 29.4 + assert out["point_power_kw"] == 29.4 + assert out["system_tps_per_kw"] == 100.0 + assert out["mlc_estimated_power"] is False + assert out["problems"] == [] + + def test_point_scaled_by_nodes_used(self) -> None: + point = {"nodes_used": [{"system_node_ensemble_id": 0, "nodes": 1}]} + out = derive_point_power(_power(), point, {}, _SUMMARY) + assert out["provisioned_power_kw"] == 29.4 + assert out["point_power_kw"] == 14.7 + assert out["system_tps_per_kw"] == 200.0 + + def test_appendix_d_default_sets_the_estimated_flag(self) -> None: + power = _power() + cpu = power["node_sets"][0]["components"]["cpu"] + cpu["tdp_per_unit"] = {"value_w": 350, "source_type": "mlc_default", "source": "D.2"} + out = derive_point_power(power, {}, {}, _SUMMARY) + assert out["mlc_estimated_power"] is True + assert out["estimated"] + + def test_submitter_declared_estimate_is_kept(self) -> None: + out = derive_point_power(_power(mlc_estimated_power=True), {}, {}, _SUMMARY) + assert out["mlc_estimated_power"] is True + + def test_nodes_used_beyond_provisioned_derives_no_point_figure(self) -> None: + point = {"nodes_used": [{"system_node_ensemble_id": 0, "nodes": 3}]} + out = derive_point_power(_power(), point, {}, _SUMMARY) + assert out["provisioned_power_kw"] == 29.4 + assert out["point_power_kw"] is None + assert out["system_tps_per_kw"] is None + assert out["problems"] + + def test_unknown_ensemble_derives_no_point_figure(self) -> None: + point = {"nodes_used": [{"system_node_ensemble_id": 7, "nodes": 1}]} + out = derive_point_power(_power(), point, {}, _SUMMARY) + assert out["point_power_kw"] is None + + def test_invalid_descriptor_never_raises(self) -> None: + out = derive_point_power({"cooling": "steam"}, {}, {}, _SUMMARY) + assert out["provisioned_power_kw"] is None + assert out["problems"] + + def test_missing_throughput_leaves_tps_per_kw_unset(self) -> None: + out = derive_point_power(_power(), {}, {}, {}) + assert out["point_power_kw"] == 29.4 + assert out["system_tps_per_kw"] is None + + def test_inputs_not_mutated(self) -> None: + power = _power() + before = copy.deepcopy(power) + derive_point_power(power, {}, {}, _SUMMARY) + assert power == before + + +@pytest.mark.unit +class TestHostCoresByEnsemble: + def test_reads_node_types_and_coerces_ids(self) -> None: + desc = { + "node_types": [ + {"system_node_ensemble_id": "0", "host_processor_core_count": 64}, + {"system_node_ensemble_id": 1, "host_processor_core_count": 32}, + {"system_node_ensemble_id": 2}, + "junk", + ] + } + assert host_cores_by_ensemble(desc) == {0: 64, 1: 32}