diff --git a/MODULE.bazel b/MODULE.bazel index 2fcc25c65..75ebf0260 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ module( ) bazel_dep(name = "bazel_features", version = "1.38.0") -bazel_dep(name = "bazel_lib", version = "3.0.0") +bazel_dep(name = "bazel_lib", version = "3.2.0") bazel_dep(name = "bazel_skylib", version = "1.4.2") bazel_dep(name = "gawk", version = "5.3.2.bcr.7") bazel_dep(name = "platforms", version = "1.0.0") diff --git a/docs/uv-patching.md b/docs/uv-patching.md index 0f3fb4dd6..033f6fa33 100644 --- a/docs/uv-patching.md +++ b/docs/uv-patching.md @@ -83,6 +83,23 @@ Pre-build patches are applied to the extracted source tree after archive extract - Removing problematic native build dependencies - Patching source code that affects the build output +### Measuring wheel build memory + +On Linux, when `resource_set` is set, rules_py samples the aggregate resident +memory of the build process and its descendants and reports the peak to stderr: + +```starlark +uv.override_package( + lock = "//:uv.lock", + name = "native-package", + resource_set = "mem_2g", +) +``` + +A measurement appears whenever the sampled peak crosses another 256 MiB +threshold and once when the build finishes. If procfs is unavailable or cannot +be read, the final measurement is reported as unavailable. + ### Adding extra dependencies or data Some packages have implicit runtime dependencies that aren't declared in their metadata: diff --git a/e2e/BUILD.bazel b/e2e/BUILD.bazel index 28d2a93a9..be5da9667 100644 --- a/e2e/BUILD.bazel +++ b/e2e/BUILD.bazel @@ -110,13 +110,12 @@ write_source_files( # shows up as the per-wheel dicts collapsing into one merged list. "snapshots/abi3_compat.whl_install.cffi.BUILD.bazel": "@whl_install__abi3_compat__cffi__2_0_0//:BUILD.bazel", - # @sdist_build for jpype1 with uv.override_package toolchains/env + # @sdist_build for jpype1 with uv.override_package toolchains and env # ---------------------------------------------------------------- # Covers the override_package -> repo-rule -> BUILD-template - # plumbing for the augment-on-defaults `toolchains = [...]` / - # `env = {...}`. The override layers a JDK runtime on top of the - # default CC toolchain; the snapshot pins both toolchain - # references and both env families (CC + JAVA_HOME/JAVA/JAR). + # plumbing for `toolchains = [...]` and `env = {...}`. The override + # layers a JDK runtime on top of the default CC toolchain; the snapshot + # pins the toolchain references and both env families (CC + JAVA_HOME/JAVA/JAR). # Pairs with the analysis test at # //uv/private/pep517_whl:toolchain_env_test which asserts the # rule expands env keys correctly given an explicit fixture. diff --git a/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel b/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel index 83f105fb0..ece40aa71 100644 --- a/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel +++ b/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel @@ -23,9 +23,8 @@ uv.unstable_annotate_packages( lock = "//cases/uv-sdist-jdk-build:uv.lock", ) -# Layer a JDK runtime toolchain into the build env. `toolchains` and -# `env` here are additive — these entries are appended to whatever -# sdist_build's BUILD template emits by default. +# Layer a JDK runtime toolchain into the build env. +# `toolchains` and `env` are additive to sdist_build's defaults. uv.override_package( name = "jpype1", env = { diff --git a/e2e/cases/uv-sdist-mpicc/BUILD.bazel b/e2e/cases/uv-sdist-mpicc/BUILD.bazel index 46a439565..8224e48c5 100644 --- a/e2e/cases/uv-sdist-mpicc/BUILD.bazel +++ b/e2e/cases/uv-sdist-mpicc/BUILD.bazel @@ -9,7 +9,10 @@ load("@aspect_rules_py//py:defs.bzl", "py_test") py_test( name = "test", srcs = ["__test__.py"], - data = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], + data = [ + "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", + "@aspect_rules_py//uv/private/pep517_whl:memory_monitor", + ], dep_group = "uv_sdist_mpicc", main = "__test__.py", python_version = "3.11", diff --git a/e2e/cases/uv-sdist-mpicc/__test__.py b/e2e/cases/uv-sdist-mpicc/__test__.py index c10027653..192664d50 100644 --- a/e2e/cases/uv-sdist-mpicc/__test__.py +++ b/e2e/cases/uv-sdist-mpicc/__test__.py @@ -76,10 +76,18 @@ def make_sdist(workdir): def run_helper(helper, sdist, outdir, path_entries, expect): + from pathlib import Path as _Path + + helper_path = _Path(helper) + assert helper_path.parts[-4:] == ("uv", "private", "pep517_whl", "build_helper.py"), ( + "Unexpected helper path structure: {}".format(helper) + ) + rules_py_root = str(helper_path.parents[3]) env = { "MPICC_TEST_EXPECT": expect, "PATH": os.pathsep.join(path_entries), "HOME": os.environ.get("TEST_TMPDIR", "/tmp"), + "PYTHONPATH": rules_py_root, } cc = shutil.which("cc") or "/usr/bin/cc" if os.path.exists(cc): diff --git a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel index daa114fdc..f61aedd6b 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel @@ -6,7 +6,7 @@ py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:scikit_build_core", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:setuptools", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:wheel", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:build", "@whl_install__uv_sdist_jdk_build__scikit_build_core__0_12_2//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:scikit_build_core", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:setuptools", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:wheel", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:build", "@whl_install__uv_sdist_jdk_build__scikit_build_core__0_12_2//:install"], ) pep517_native_whl( diff --git a/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel index 6cbd37583..505501e11 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel @@ -6,7 +6,7 @@ py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@@aspect_rules_py++uv+project__uv_sdist_native_build//:build", "@@aspect_rules_py++uv+project__uv_sdist_native_build//:setuptools", "@whl_install__uv_sdist_native_build__setuptools__75_8_2//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@aspect_rules_py++uv+project__uv_sdist_native_build//:build", "@@aspect_rules_py++uv+project__uv_sdist_native_build//:setuptools", "@whl_install__uv_sdist_native_build__setuptools__75_8_2//:install"], ) pep517_native_whl( diff --git a/uv/private/extension/defs.bzl b/uv/private/extension/defs.bzl index 38584211e..08494ce53 100644 --- a/uv/private/extension/defs.bzl +++ b/uv/private/extension/defs.bzl @@ -770,8 +770,8 @@ _override_package_tag = tag_class( "bucket.", ), - # Per-package toolchain plumbing for native sdist builds. Both - # attributes AUGMENT the defaults baked into sdist_build's + # Per-package resource and toolchain settings for sdist builds. + # toolchains and env AUGMENT the defaults baked into sdist_build's # generated `pep517_native_whl(...)` call (the CC toolchain + # CC/CXX/AR/LD/STRIP env) — they don't replace them. Use these # to layer extra toolchains (Java runtime, Rust, …) and extra @@ -784,7 +784,6 @@ _override_package_tag = tag_class( default = {}, doc = "Extra environment variables merged into the build action's `env` dict. Values may reference $(VAR) make-variables sourced from the default CC toolchain or any extra `toolchains` listed above.", ), - # Pre-build patches: applied to extracted sdist source before wheel build. "pre_build_patches": attr.label_list( default = [], diff --git a/uv/private/pep517_whl/BUILD.bazel b/uv/private/pep517_whl/BUILD.bazel index 5b52d993a..26a84e5bf 100644 --- a/uv/private/pep517_whl/BUILD.bazel +++ b/uv/private/pep517_whl/BUILD.bazel @@ -1,8 +1,12 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load(":rule.bzl", "pep517_native_whl") -load(":test.bzl", "pep517_native_whl_toolchain_env_test") +load("//py:defs.bzl", "py_library", "py_test") +load(":rule.bzl", "pep517_native_whl", "pep517_whl") +load( + ":test.bzl", + "pep517_native_whl_toolchain_env_test", +) package(default_visibility = [ "//uv/private:__subpackages__", @@ -22,6 +26,19 @@ bzl_library( ], ) +py_library( + name = "memory_monitor", + srcs = ["memory_monitor.py"], + visibility = ["//visibility:public"], +) + +py_test( + name = "memory_monitor_test", + srcs = ["memory_monitor_test.py"], + main = "memory_monitor_test.py", + deps = [":memory_monitor"], +) + bzl_library( name = "test", srcs = ["test.bzl"], @@ -91,3 +108,8 @@ pep517_native_whl_toolchain_env_test( name = "toolchain_env_test", target_under_test = ":__toolchain_env_fixture", ) + +bzl_library( + name = "build_memory", + srcs = ["build_memory.bzl"], +) diff --git a/uv/private/pep517_whl/build_helper.py b/uv/private/pep517_whl/build_helper.py index 10774c6c3..8a5e9d3dc 100644 --- a/uv/private/pep517_whl/build_helper.py +++ b/uv/private/pep517_whl/build_helper.py @@ -13,9 +13,11 @@ import shutil import sys from os import chmod, defpath, listdir, makedirs, path, pathsep -from subprocess import CalledProcessError, check_call, check_output, STDOUT, run +from subprocess import CalledProcessError, check_call, check_output, STDOUT from tempfile import TemporaryFile +from uv.private.pep517_whl.memory_monitor import run_with_memory_profile + try: import tomllib except ModuleNotFoundError: @@ -187,6 +189,7 @@ def _legacy_metadata_conflicts_with_pyproject(worktree): PARSER = ArgumentParser() PARSER.add_argument("srcarchive") PARSER.add_argument("outdir") +PARSER.add_argument("--monitor-memory", action="store_true") PARSER.add_argument("--validate-anyarch", action="store_true") PARSER.add_argument("--patch-strip", type=int, default=0, help="Strip count for patch (-p)") PARSER.add_argument("--patch", action="append", default=[], dest="patches", help="Patch file to apply (repeatable)") @@ -279,7 +282,14 @@ def _legacy_metadata_conflicts_with_pyproject(worktree): with TemporaryFile(mode="w+") as build_log: try: - run(cmd, cwd=t, env=build_env, stdout=build_log, stderr=STDOUT, check=True) + run_with_memory_profile( + cmd, + cwd=t, + env=build_env, + stdout=build_log, + wheel=path.basename(opts.srcarchive), + monitor=opts.monitor_memory, + ) except CalledProcessError: build_log.seek(0) output = build_log.read() diff --git a/uv/private/pep517_whl/build_memory.bzl b/uv/private/pep517_whl/build_memory.bzl new file mode 100644 index 000000000..17297ede9 --- /dev/null +++ b/uv/private/pep517_whl/build_memory.bzl @@ -0,0 +1,12 @@ +"""Validation shared by wheel build configuration surfaces.""" + +# bazel-lib 3.2 clamps larger resource requests to 32 GiB: +# https://github.com/aspect-build/bazel-lib/blob/v3.2.0/lib/private/resource_sets.bzl#L2224-L2227 +MAX_BUILD_MEMORY_MB = 32768 + +def validate_build_memory_mb(value, owner): + if value < 0 or value > MAX_BUILD_MEMORY_MB: + fail("{}: build_memory_mb must be between 0 and {}".format( + owner, + MAX_BUILD_MEMORY_MB, + )) diff --git a/uv/private/pep517_whl/memory_monitor.py b/uv/private/pep517_whl/memory_monitor.py new file mode 100644 index 000000000..06736f845 --- /dev/null +++ b/uv/private/pep517_whl/memory_monitor.py @@ -0,0 +1,168 @@ +"""Run a wheel build while reporting its aggregate process-tree memory.""" + +import os +import signal +import subprocess +import sys +import time + +_MIB = 1024 * 1024 +_REPORT_STEP_BYTES = 256 * _MIB +_SAMPLE_INTERVAL_SECONDS = 0.25 + + +def _exit_on_sigterm(signum, _frame): + raise SystemExit(128 + signum) + + +_children_unavailable_warned = False + + +def _process_tree_rss_bytes(root_pid, page_size, proc_root="/proc"): + """Return aggregate RSS for root_pid and its descendants on Linux.""" + global _children_unavailable_warned + # statm reports resident memory in pages: + # https://man7.org/linux/man-pages/man5/proc_pid_statm.5.html + # + # Each task's children file lists only immediate children, so recurse: + # https://docs.kernel.org/filesystems/proc.html#proc-pid-task-tid-children-information-about-task-children + pending = [root_pid] + seen = set() + total = 0 + while pending: + pid = pending.pop() + if pid in seen: + continue + seen.add(pid) + process_dir = os.path.join(proc_root, str(pid)) + + try: + with open(os.path.join(process_dir, "statm")) as statm: + total += int(statm.read().split()[1]) * page_size + task_ids = os.listdir(os.path.join(process_dir, "task")) + except ( + FileNotFoundError, + PermissionError, + ProcessLookupError, + ValueError, + IndexError, + ): + continue + + any_children_file_found = False + for task_id in task_ids: + try: + with open( + os.path.join(process_dir, "task", task_id, "children") + ) as children: + pending.extend(int(child) for child in children.read().split()) + any_children_file_found = True + except FileNotFoundError: + pass # children file absent; may indicate missing CONFIG_PROC_CHILDREN + except (PermissionError, ProcessLookupError, ValueError): + any_children_file_found = True # file exists; transient access error + continue + + if task_ids and not any_children_file_found and not _children_unavailable_warned: + _children_unavailable_warned = True + print( + "rules_py: /proc/{pid}/task/{tid}/children is unavailable; " + "memory sampling covers only the root process " + "(kernel may lack CONFIG_PROC_CHILDREN).", + file=sys.stderr, + flush=True, + ) + + return total + + +def _report_memory(wheel, state, current, peak): + if peak is None: + details = "unavailable" + elif current is None: + details = "sampled aggregate peak={:.1f} MiB".format(peak / _MIB) + else: + details = "sampled aggregate current={:.1f} MiB, peak={:.1f} MiB".format( + current / _MIB, + peak / _MIB, + ) + print( + "rules_py wheel build memory for {} {}: {}".format( + wheel, + state, + details, + ), + file=sys.stderr, + flush=True, + ) + + +def run_with_memory_profile(cmd, *, cwd, env, stdout, wheel, monitor): + """Run cmd, optionally reporting process-tree memory.""" + _sp_kwargs = {"cwd": cwd, "env": env, "stdout": stdout, "stderr": subprocess.STDOUT} + + if not monitor: + subprocess.run(cmd, check=True, **_sp_kwargs) + return + + can_sample_proc = sys.platform.startswith("linux") and os.path.isdir("/proc") + try: + page_size = os.sysconf("SC_PAGE_SIZE") if can_sample_proc else None + except (OSError, ValueError): + can_sample_proc = False + page_size = None + + if not can_sample_proc: + try: + subprocess.run(cmd, check=True, **_sp_kwargs) + except subprocess.CalledProcessError: + _report_memory(wheel, "finished", None, None) + raise + _report_memory(wheel, "finished", None, None) + return + + previous_sigterm_handler = signal.signal(signal.SIGTERM, _exit_on_sigterm) + process = None + + try: + process = subprocess.Popen( + cmd, + start_new_session=True, + **_sp_kwargs, + ) + peak = 0 + next_report = _REPORT_STEP_BYTES + + while True: + if can_sample_proc: + try: + current = _process_tree_rss_bytes(process.pid, page_size) + except (OSError, ValueError): + can_sample_proc = False + peak = None + else: + peak = max(peak, current) + if peak >= next_report: + _report_memory(wheel, "running", current, peak) + next_report = ( + peak // _REPORT_STEP_BYTES + 1 + ) * _REPORT_STEP_BYTES + + returncode = process.poll() + if returncode is not None: + break + time.sleep(_SAMPLE_INTERVAL_SECONDS) + except BaseException: + if process is not None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + raise + finally: + signal.signal(signal.SIGTERM, previous_sigterm_handler) + + _report_memory(wheel, "finished", None, peak or None) + if returncode: + raise subprocess.CalledProcessError(returncode, cmd) diff --git a/uv/private/pep517_whl/memory_monitor_test.py b/uv/private/pep517_whl/memory_monitor_test.py new file mode 100644 index 000000000..776eea555 --- /dev/null +++ b/uv/private/pep517_whl/memory_monitor_test.py @@ -0,0 +1,219 @@ +import contextlib +import io +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + +from uv.private.pep517_whl import memory_monitor + + +class ProcessTreeRssTest(unittest.TestCase): + def test_sums_descendants_spawned_by_any_task(self): + with tempfile.TemporaryDirectory() as directory: + proc_root = Path(directory) + self._write_process(proc_root, 10, 2, {10: [20], 11: [30]}) + self._write_process(proc_root, 20, 3, {20: []}) + self._write_process(proc_root, 30, 5, {30: []}) + + self.assertEqual( + 10 * 4096, + memory_monitor._process_tree_rss_bytes(10, 4096, str(proc_root)), + ) + + def test_missing_children_file_warns_and_covers_only_root(self): + """When children files are absent (CONFIG_PROC_CHILDREN missing), the walk + stops at the root PID and emits a one-time warning to stderr.""" + stderr = io.StringIO() + with tempfile.TemporaryDirectory() as directory: + proc_root = Path(directory) + + process_dir = proc_root / "10" + process_dir.mkdir() + (process_dir / "statm").write_text("100 5 0 0 0 0 0\n") + (process_dir / "task" / "10").mkdir(parents=True) + child_dir = proc_root / "20" + child_dir.mkdir() + (child_dir / "statm").write_text("100 3 0 0 0 0 0\n") + + with ( + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor, "_children_unavailable_warned", False), + ): + result = memory_monitor._process_tree_rss_bytes(10, 4096, str(proc_root)) + + self.assertEqual(5 * 4096, result) # only root PID; process 20 unreachable + self.assertIn("CONFIG_PROC_CHILDREN", stderr.getvalue()) + + def test_success_without_monitoring_uses_run(self): + with tempfile.TemporaryFile() as stdout: + memory_monitor.run_with_memory_profile( + [sys.executable, "-c", "pass"], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=False, + ) + + def test_failed_child_reports_memory_before_raising(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor, "_REPORT_STEP_BYTES", 1), + ): + with self.assertRaisesRegex(subprocess.CalledProcessError, "exit status 7"): + memory_monitor.run_with_memory_profile( + [ + sys.executable, + "-c", + "import time; data = bytearray(16 * 1024 * 1024); time.sleep(.5); raise SystemExit(7)", + ], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + report = stderr.getvalue() + if sys.platform.startswith("linux") and os.path.isdir("/proc"): + self.assertRegex( + report, + r"wheel build memory for test-wheel[.]tar[.]gz running: sampled aggregate current=[1-9][0-9.]* MiB, peak=[1-9][0-9.]* MiB", + ) + self.assertRegex( + report, + r"wheel build memory for test-wheel[.]tar[.]gz finished: sampled aggregate peak=[1-9][0-9.]* MiB", + ) + else: + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + report, + ) + + def test_sampler_failure_reports_unavailable(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor.sys, "platform", "linux"), + mock.patch.object(memory_monitor.os.path, "isdir", return_value=True), + mock.patch.object( + memory_monitor, + "_process_tree_rss_bytes", + side_effect=OSError("procfs unavailable"), + ), + ): + memory_monitor.run_with_memory_profile( + [sys.executable, "-c", "import time; time.sleep(.1)"], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + stderr.getvalue(), + ) + + def test_unsupported_platform_uses_run(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor.sys, "platform", "win32"), + mock.patch.object(memory_monitor.subprocess, "run") as run, + ): + memory_monitor.run_with_memory_profile( + ["command"], + cwd=None, + env={}, + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + run.assert_called_once() + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + stderr.getvalue(), + ) + + @unittest.skipUnless( + sys.platform.startswith("linux") and os.path.isdir("/proc"), + "requires Linux procfs", + ) + def test_exception_kills_and_waits_for_process_tree(self): + for interrupt_signal, expected_exception in ( + (signal.SIGINT, KeyboardInterrupt), + (signal.SIGTERM, SystemExit), + ): + with ( + self.subTest(signal=interrupt_signal), + tempfile.TemporaryDirectory() as directory, + tempfile.TemporaryFile() as stdout, + ): + pids_file = Path(directory) / "pids" + with self.assertRaises(expected_exception) as raised: + memory_monitor.run_with_memory_profile( + [ + sys.executable, + "-c", + "import os, pathlib, signal, subprocess, sys, time; " + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)']); " + "pathlib.Path(sys.argv[1]).write_text(f'{os.getpid()} {child.pid}'); " + "os.kill(os.getppid(), int(sys.argv[2])); " + "time.sleep(60)", + str(pids_file), + str(int(interrupt_signal)), + ], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + if interrupt_signal == signal.SIGTERM: + self.assertEqual(128 + signal.SIGTERM, raised.exception.code) + + frontend_pid, child_pid = [ + int(pid) for pid in pids_file.read_text().split() + ] + for pid in (frontend_pid, child_pid): + deadline = time.monotonic() + 5 + while True: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + if time.monotonic() >= deadline: + self.fail( + "process {} survived interruption cleanup".format(pid) + ) + time.sleep(0.01) + + @staticmethod + def _write_process(proc_root, pid, rss_pages, task_children): + process_dir = proc_root / str(pid) + process_dir.mkdir() + (process_dir / "statm").write_text("100 {} 0 0 0 0 0\n".format(rss_pages)) + for task_id, children in task_children.items(): + task_dir = process_dir / "task" / str(task_id) + task_dir.mkdir(parents=True) + (task_dir / "children").write_text( + " ".join(str(child) for child in children) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv/private/pep517_whl/rule.bzl b/uv/private/pep517_whl/rule.bzl index 1fb57149e..40ff8e0aa 100644 --- a/uv/private/pep517_whl/rule.bzl +++ b/uv/private/pep517_whl/rule.bzl @@ -29,6 +29,9 @@ def _patch_args_and_inputs(ctx): patch_inputs.append(f) return patch_args, patch_inputs +def _memory_args(ctx): + return ["--monitor-memory"] if ctx.attr.resource_set != "default" else [] + def _collect_toolchain_inputs_and_vars(ctx): """Gather files + Make-variable substitutions from `ctx.attr.toolchains`. @@ -97,7 +100,7 @@ def _cc_toolchain_inputs_and_compiler(ctx): def _pep517_whl(ctx): archive = ctx.file.src - wheel_dir = ctx.actions.declare_directory("whl") + wheel_dir = ctx.actions.declare_directory(ctx.label.name) patch_args, patch_inputs = _patch_args_and_inputs(ctx) # The build tool is a py_binary wrapping build_helper.py. Using it as @@ -110,7 +113,7 @@ def _pep517_whl(ctx): progress_message = "Source compiling {} to a whl".format(archive.basename), executable = ctx.executable.tool, toolchain = None, - arguments = ctx.attr.args + patch_args + [ + arguments = ctx.attr.args + patch_args + _memory_args(ctx) + [ archive.path, wheel_dir.path, ], @@ -126,7 +129,7 @@ def _pep517_whl(ctx): def _pep517_native_whl(ctx): archive = ctx.file.src - wheel_dir = ctx.actions.declare_directory("whl") + wheel_dir = ctx.actions.declare_directory(ctx.label.name) patch_args, patch_inputs = _patch_args_and_inputs(ctx) env = _common_env(ctx) @@ -148,7 +151,7 @@ def _pep517_native_whl(ctx): progress_message = "Native source compiling {} to a whl".format(archive.basename), executable = ctx.executable.tool, toolchain = None, - arguments = ctx.attr.args + patch_args + [ + arguments = ctx.attr.args + patch_args + _memory_args(ctx) + [ archive.path, wheel_dir.path, ], diff --git a/uv/private/pep517_whl/test.bzl b/uv/private/pep517_whl/test.bzl index 9f9b758cb..510a6a900 100644 --- a/uv/private/pep517_whl/test.bzl +++ b/uv/private/pep517_whl/test.bzl @@ -17,6 +17,26 @@ _JDK_ENV_KEYS = ["JAVA_HOME", "JAVA", "JAR"] _REQUIRED_ENV_KEYS = _CC_ENV_KEYS + _JDK_ENV_KEYS +def _analysis_failure_test_impl(ctx): + env = analysistest.begin(ctx) + asserts.expect_failure(env, ctx.attr.expected_error) + return analysistest.end(env) + +build_memory_failure_test = analysistest.make( + _analysis_failure_test_impl, + attrs = {"expected_error": attr.string()}, + expect_failure = True, +) + +def _build_memory_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + actions = [action for action in target.actions if action.mnemonic == "PySdistBuild"] + asserts.equals(env, 1, len(actions)) + return analysistest.end(env) + +build_memory_test = analysistest.make(_build_memory_test_impl) + def _toolchain_env_test_impl(ctx): env = analysistest.begin(ctx) target = analysistest.target_under_test(env) diff --git a/uv/private/sdist_build/repository.bzl b/uv/private/sdist_build/repository.bzl index 8ceb77b3c..1be47608c 100644 --- a/uv/private/sdist_build/repository.bzl +++ b/uv/private/sdist_build/repository.bzl @@ -208,6 +208,9 @@ def _sdist_build_impl(repository_ctx): # Merge explicit deps with auto-discovered deps all_deps = [str(d) for d in repository_ctx.attr.deps] + extra_dep_labels + build_tool_deps = [ + "@aspect_rules_py//uv/private/pep517_whl:memory_monitor", + ] + all_deps pre_build_patches = repository_ctx.attr.pre_build_patches patch_attrs = "" @@ -285,7 +288,7 @@ exports_files( ) """.format( src = repository_ctx.attr.src, - deps = repr(all_deps), + deps = repr(build_tool_deps), rule = "pep517_native_whl" if is_native else "pep517_whl", version = repository_ctx.attr.version, resource_set_attr = resource_set_attr, diff --git a/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel b/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel index d8ca27afe..54d448b05 100644 --- a/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel +++ b/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel @@ -6,7 +6,7 @@ py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@@+uv+project__aspect_rules_py//:build", "@@+uv+project__aspect_rules_py//:setuptools", "@@+uv+project__aspect_rules_py//:maturin", "@whl_install__aspect_rules_py__setuptools__80_9_0//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@+uv+project__aspect_rules_py//:build", "@@+uv+project__aspect_rules_py//:setuptools", "@@+uv+project__aspect_rules_py//:maturin", "@whl_install__aspect_rules_py__setuptools__80_9_0//:install"], ) pep517_whl(