From f1048c6550d3d405524596044fd9e0402e6efd3a Mon Sep 17 00:00:00 2001 From: xangcastle Date: Mon, 22 Jun 2026 21:50:26 -0600 Subject: [PATCH 1/6] xangcastle/uv-1-wheel-build-memory --- MODULE.bazel | 2 +- docs/uv-patching.md | 22 ++ e2e/BUILD.bazel | 9 +- .../uv-sdist-jdk-build/setup.MODULE.bazel | 6 +- e2e/cases/uv-sdist-mpicc/BUILD.bazel | 5 +- e2e/cases/uv-sdist-mpicc/__test__.py | 8 + ...uild.uv_sdist_jdk_build.jpype1.BUILD.bazel | 3 +- uv/private/extension/BUILD.bazel | 1 + uv/private/extension/defs.bzl | 26 ++- uv/private/pep517_whl/BUILD.bazel | 63 ++++- uv/private/pep517_whl/build_helper.py | 14 +- uv/private/pep517_whl/build_memory.bzl | 12 + uv/private/pep517_whl/memory_monitor.py | 168 ++++++++++++++ uv/private/pep517_whl/memory_monitor_test.py | 219 ++++++++++++++++++ uv/private/pep517_whl/rule.bzl | 25 +- uv/private/pep517_whl/test.bzl | 20 ++ uv/private/sdist_build/repository.bzl | 16 +- .../sdist_build.bravado_core.BUILD.bazel | 2 +- 18 files changed, 593 insertions(+), 28 deletions(-) create mode 100644 uv/private/pep517_whl/build_memory.bzl create mode 100644 uv/private/pep517_whl/memory_monitor.py create mode 100644 uv/private/pep517_whl/memory_monitor_test.py diff --git a/MODULE.bazel b/MODULE.bazel index 2fcc25c65..75ebf0260 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ module( ) bazel_dep(name = "bazel_features", version = "1.38.0") -bazel_dep(name = "bazel_lib", version = "3.0.0") +bazel_dep(name = "bazel_lib", version = "3.2.0") bazel_dep(name = "bazel_skylib", version = "1.4.2") bazel_dep(name = "gawk", version = "5.3.2.bcr.7") bazel_dep(name = "platforms", version = "1.0.0") diff --git a/docs/uv-patching.md b/docs/uv-patching.md index b67d02c85..483633da9 100644 --- a/docs/uv-patching.md +++ b/docs/uv-patching.md @@ -83,6 +83,28 @@ Pre-build patches are applied to the extracted source tree after archive extract - Removing problematic native build dependencies - Patching source code that affects the build output +### Allocating wheel build memory + +Set `build_memory_mb` to the expected peak memory of a local sdist build so +Bazel can limit concurrent wheel builds: + +```starlark +uv.override_package( + lock = "//:uv.lock", + name = "native-package", + build_memory_mb = 6144, +) +``` + +Bazel rounds the estimate up to one of its supported resource classes. On +Linux, setting a nonzero estimate also enables aggregate resident-memory +sampling for the build process and its descendants. Estimates above 32768 MiB +are rejected because bazel-lib 3.2 would otherwise clamp them to 32768 MiB and +silently under-reserve the action. A measurement appears whenever the sampled +peak crosses another 256 MiB threshold and once when the build finishes. If +procfs is unavailable or cannot be read, the final measurement is reported as +unavailable. + ### Adding extra dependencies or data Some packages have implicit runtime dependencies that aren't declared in their metadata: diff --git a/e2e/BUILD.bazel b/e2e/BUILD.bazel index 9217b9fad..1d46c4d34 100644 --- a/e2e/BUILD.bazel +++ b/e2e/BUILD.bazel @@ -110,13 +110,14 @@ write_source_files( # shows up as the per-wheel dicts collapsing into one merged list. "snapshots/abi3_compat.whl_install.cffi.BUILD.bazel": "@whl_install__abi3_compat__cffi__2_0_0//:BUILD.bazel", - # @sdist_build for jpype1 with uv.override_package toolchains/env + # @sdist_build for jpype1 with uv.override_package build resources, + # toolchains, and env # ---------------------------------------------------------------- # Covers the override_package -> repo-rule -> BUILD-template - # plumbing for the augment-on-defaults `toolchains = [...]` / + # plumbing for `build_memory_mb`, `toolchains = [...]`, and # `env = {...}`. The override layers a JDK runtime on top of the - # default CC toolchain; the snapshot pins both toolchain - # references and both env families (CC + JAVA_HOME/JAVA/JAR). + # default CC toolchain; the snapshot pins the memory declaration, + # toolchain references, and both env families (CC + JAVA_HOME/JAVA/JAR). # Pairs with the analysis test at # //uv/private/pep517_whl:toolchain_env_test which asserts the # rule expands env keys correctly given an explicit fixture. diff --git a/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel b/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel index 83f105fb0..e696558bc 100644 --- a/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel +++ b/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel @@ -23,11 +23,11 @@ uv.unstable_annotate_packages( lock = "//cases/uv-sdist-jdk-build:uv.lock", ) -# Layer a JDK runtime toolchain into the build env. `toolchains` and -# `env` here are additive — these entries are appended to whatever -# sdist_build's BUILD template emits by default. +# Reserve local build memory and layer a JDK runtime toolchain into the build +# env. `toolchains` and `env` are additive to sdist_build's defaults. uv.override_package( name = "jpype1", + build_memory_mb = 2048, env = { "JAR": "$(JAVABASE)/bin/jar", "JAVA": "$(JAVA)", diff --git a/e2e/cases/uv-sdist-mpicc/BUILD.bazel b/e2e/cases/uv-sdist-mpicc/BUILD.bazel index 46a439565..8224e48c5 100644 --- a/e2e/cases/uv-sdist-mpicc/BUILD.bazel +++ b/e2e/cases/uv-sdist-mpicc/BUILD.bazel @@ -9,7 +9,10 @@ load("@aspect_rules_py//py:defs.bzl", "py_test") py_test( name = "test", srcs = ["__test__.py"], - data = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], + data = [ + "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", + "@aspect_rules_py//uv/private/pep517_whl:memory_monitor", + ], dep_group = "uv_sdist_mpicc", main = "__test__.py", python_version = "3.11", diff --git a/e2e/cases/uv-sdist-mpicc/__test__.py b/e2e/cases/uv-sdist-mpicc/__test__.py index c10027653..192664d50 100644 --- a/e2e/cases/uv-sdist-mpicc/__test__.py +++ b/e2e/cases/uv-sdist-mpicc/__test__.py @@ -76,10 +76,18 @@ def make_sdist(workdir): def run_helper(helper, sdist, outdir, path_entries, expect): + from pathlib import Path as _Path + + helper_path = _Path(helper) + assert helper_path.parts[-4:] == ("uv", "private", "pep517_whl", "build_helper.py"), ( + "Unexpected helper path structure: {}".format(helper) + ) + rules_py_root = str(helper_path.parents[3]) env = { "MPICC_TEST_EXPECT": expect, "PATH": os.pathsep.join(path_entries), "HOME": os.environ.get("TEST_TMPDIR", "/tmp"), + "PYTHONPATH": rules_py_root, } cc = shutil.which("cc") or "/usr/bin/cc" if os.path.exists(cc): diff --git a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel index daa114fdc..4e25a55c9 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel @@ -6,7 +6,7 @@ py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:scikit_build_core", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:setuptools", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:wheel", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:build", "@whl_install__uv_sdist_jdk_build__scikit_build_core__0_12_2//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:scikit_build_core", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:setuptools", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:wheel", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:build", "@whl_install__uv_sdist_jdk_build__scikit_build_core__0_12_2//:install"], ) pep517_native_whl( @@ -15,6 +15,7 @@ pep517_native_whl( tool = ":build_tool", version = "1.7.1", args = [], + build_memory_mb = 2048, toolchains = [ "@bazel_tools//tools/cpp:current_cc_toolchain", "@@bazel_tools//tools/jdk:current_java_runtime", diff --git a/uv/private/extension/BUILD.bazel b/uv/private/extension/BUILD.bazel index 05b675745..97e11353a 100644 --- a/uv/private/extension/BUILD.bazel +++ b/uv/private/extension/BUILD.bazel @@ -22,6 +22,7 @@ bzl_library( "//uv/private:normalize_version", "//uv/private/constraints:repository", "//uv/private/git_archive:repository", + "//uv/private/pep517_whl:build_memory", "//uv/private/pprint:defs", "//uv/private/sdist_build:repository", "//uv/private/sdist_configure:defs", diff --git a/uv/private/extension/defs.bzl b/uv/private/extension/defs.bzl index 6c0b16e10..440d283f6 100644 --- a/uv/private/extension/defs.bzl +++ b/uv/private/extension/defs.bzl @@ -60,6 +60,7 @@ load("//uv/private:normalize_version.bzl", "normalize_version") load("//uv/private:parse_whl_name.bzl", "parse_whl_name") load("//uv/private/constraints:repository.bzl", "configurations_hub") load("//uv/private/git_archive:repository.bzl", "git_archive") +load("//uv/private/pep517_whl:build_memory.bzl", "validate_build_memory_mb") load("//uv/private/pprint:defs.bzl", "pprint") load("//uv/private/sdist_build:repository.bzl", "sdist_build") load("//uv/private/sdist_configure:defs.bzl", "DEFAULT_CONFIGURE_SCRIPT") @@ -246,20 +247,25 @@ def _parse_projects(module_ctx, hub_specs): )) has_target = override.target != None + validate_build_memory_mb( + override.build_memory_mb, + "uv.override_package() for '{}'".format(override.name), + ) has_modifications = ( override.pre_build_patches or override.post_install_patches or override.extra_deps or override.extra_data or override.toolchains or - override.env + override.env or + override.build_memory_mb ) if has_target and has_modifications: - fail("uv.override_package() for '{}': `target` is mutually exclusive with patch/exclude attributes. Use `target` for full replacement OR patch/exclude attributes for modifications, not both.".format(override.name)) + fail("uv.override_package() for '{}': `target` is mutually exclusive with modification attributes. Use `target` for full replacement OR build, patch, and data attributes for modifications, not both.".format(override.name)) if not has_target and not has_modifications: - fail("uv.override_package() for '{}': must specify either `target` for full replacement or at least one modification attribute (pre_build_patches, post_install_patches, extra_deps, extra_data).".format(override.name)) + fail("uv.override_package() for '{}': must specify either `target` for full replacement or at least one build, patch, or data modification attribute.".format(override.name)) package_overrides[override_key] = override @@ -410,9 +416,11 @@ def _parse_projects(module_ctx, hub_specs): # they don't replace them. Empty == no augmentation. extra_toolchains = [] extra_env = {} + build_memory_mb = 0 if pkg_override: extra_toolchains = [str(t) for t in pkg_override.toolchains] extra_env = pkg_override.env + build_memory_mb = pkg_override.build_memory_mb sbuild_specs[sbuild_id] = struct( src = sdist, @@ -425,6 +433,7 @@ def _parse_projects(module_ctx, hub_specs): configure_command = project.unstable_configure_command, extra_toolchains = extra_toolchains, extra_env = extra_env, + build_memory_mb = build_memory_mb, ) has_sbuild = True @@ -649,6 +658,8 @@ def _uv_impl(module_ctx): sbuild_kwargs["extra_toolchains"] = sbuild_cfg.extra_toolchains if sbuild_cfg.extra_env: sbuild_kwargs["extra_env"] = sbuild_cfg.extra_env + if sbuild_cfg.build_memory_mb: + sbuild_kwargs["build_memory_mb"] = sbuild_cfg.build_memory_mb sdist_build(**sbuild_kwargs) for install_id, install_cfg in cfg.install_cfgs.items(): @@ -746,12 +757,16 @@ _override_package_tag = tag_class( # Mutually exclusive with patch/exclude attributes. "target": attr.label(mandatory = False), - # Per-package toolchain plumbing for native sdist builds. Both - # attributes AUGMENT the defaults baked into sdist_build's + # Per-package resource and toolchain settings for sdist builds. + # toolchains and env AUGMENT the defaults baked into sdist_build's # generated `pep517_native_whl(...)` call (the CC toolchain + # CC/CXX/AR/LD/STRIP env) — they don't replace them. Use these # to layer extra toolchains (Java runtime, Rust, …) and extra # env vars on top of the defaults. + "build_memory_mb": attr.int( + default = 0, + doc = "Estimated peak memory in MB for this package's local wheel build, from 0 to 32768. Bazel rounds up to a supported resource class; zero uses its default estimate.", + ), "toolchains": attr.label_list( default = [], doc = "Extra toolchain targets appended to the generated pep517_native_whl(...) call's `toolchains` list. Each target's TemplateVariableInfo make-variables become available for $(VAR) expansion in `env`.", @@ -760,7 +775,6 @@ _override_package_tag = tag_class( default = {}, doc = "Extra environment variables merged into the build action's `env` dict. Values may reference $(VAR) make-variables sourced from the default CC toolchain or any extra `toolchains` listed above.", ), - # Pre-build patches: applied to extracted sdist source before wheel build. "pre_build_patches": attr.label_list( default = [], diff --git a/uv/private/pep517_whl/BUILD.bazel b/uv/private/pep517_whl/BUILD.bazel index 62782e8d7..f5e84cb1e 100644 --- a/uv/private/pep517_whl/BUILD.bazel +++ b/uv/private/pep517_whl/BUILD.bazel @@ -1,8 +1,14 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load(":rule.bzl", "pep517_native_whl") -load(":test.bzl", "pep517_native_whl_toolchain_env_test") +load("//py:defs.bzl", "py_library", "py_test") +load(":rule.bzl", "pep517_native_whl", "pep517_whl") +load( + ":test.bzl", + "build_memory_failure_test", + "build_memory_test", + "pep517_native_whl_toolchain_env_test", +) package(default_visibility = [ "//uv/private:__subpackages__", @@ -16,7 +22,24 @@ exports_files( bzl_library( name = "rule", srcs = ["rule.bzl"], - deps = ["//py/private/toolchain:types"], + deps = [ + ":build_memory", + "//py/private/toolchain:types", + "@bazel_lib//lib:resource_sets", + ], +) + +py_library( + name = "memory_monitor", + srcs = ["memory_monitor.py"], + visibility = ["//visibility:public"], +) + +py_test( + name = "memory_monitor_test", + srcs = ["memory_monitor_test.py"], + main = "memory_monitor_test.py", + deps = [":memory_monitor"], ) bzl_library( @@ -59,6 +82,35 @@ sh_binary( tags = ["manual"], ) +pep517_whl( + name = "__excessive_memory_fixture", + src = ":__stub_sdist", + build_memory_mb = 32769, + tags = ["manual"], + tool = ":__stub_tool", + version = "0.0.1", +) + +build_memory_failure_test( + name = "build_memory_limit_test", + expected_error = "build_memory_mb must be between 0 and 32768", + target_under_test = ":__excessive_memory_fixture", +) + +pep517_whl( + name = "__maximum_memory_fixture", + src = ":__stub_sdist", + build_memory_mb = 32768, + tags = ["manual"], + tool = ":__stub_tool", + version = "0.0.1", +) + +build_memory_test( + name = "build_memory_maximum_test", + target_under_test = ":__maximum_memory_fixture", +) + # Fixture replicates the toolchains + env defaults emitted by sdist_build's # BUILD template (see uv/private/sdist_build/repository.bzl). The analysis # test below asserts these expand to non-empty toolchain paths. @@ -88,3 +140,8 @@ pep517_native_whl_toolchain_env_test( name = "toolchain_env_test", target_under_test = ":__toolchain_env_fixture", ) + +bzl_library( + name = "build_memory", + srcs = ["build_memory.bzl"], +) diff --git a/uv/private/pep517_whl/build_helper.py b/uv/private/pep517_whl/build_helper.py index 10774c6c3..8a5e9d3dc 100644 --- a/uv/private/pep517_whl/build_helper.py +++ b/uv/private/pep517_whl/build_helper.py @@ -13,9 +13,11 @@ import shutil import sys from os import chmod, defpath, listdir, makedirs, path, pathsep -from subprocess import CalledProcessError, check_call, check_output, STDOUT, run +from subprocess import CalledProcessError, check_call, check_output, STDOUT from tempfile import TemporaryFile +from uv.private.pep517_whl.memory_monitor import run_with_memory_profile + try: import tomllib except ModuleNotFoundError: @@ -187,6 +189,7 @@ def _legacy_metadata_conflicts_with_pyproject(worktree): PARSER = ArgumentParser() PARSER.add_argument("srcarchive") PARSER.add_argument("outdir") +PARSER.add_argument("--monitor-memory", action="store_true") PARSER.add_argument("--validate-anyarch", action="store_true") PARSER.add_argument("--patch-strip", type=int, default=0, help="Strip count for patch (-p)") PARSER.add_argument("--patch", action="append", default=[], dest="patches", help="Patch file to apply (repeatable)") @@ -279,7 +282,14 @@ def _legacy_metadata_conflicts_with_pyproject(worktree): with TemporaryFile(mode="w+") as build_log: try: - run(cmd, cwd=t, env=build_env, stdout=build_log, stderr=STDOUT, check=True) + run_with_memory_profile( + cmd, + cwd=t, + env=build_env, + stdout=build_log, + wheel=path.basename(opts.srcarchive), + monitor=opts.monitor_memory, + ) except CalledProcessError: build_log.seek(0) output = build_log.read() diff --git a/uv/private/pep517_whl/build_memory.bzl b/uv/private/pep517_whl/build_memory.bzl new file mode 100644 index 000000000..17297ede9 --- /dev/null +++ b/uv/private/pep517_whl/build_memory.bzl @@ -0,0 +1,12 @@ +"""Validation shared by wheel build configuration surfaces.""" + +# bazel-lib 3.2 clamps larger resource requests to 32 GiB: +# https://github.com/aspect-build/bazel-lib/blob/v3.2.0/lib/private/resource_sets.bzl#L2224-L2227 +MAX_BUILD_MEMORY_MB = 32768 + +def validate_build_memory_mb(value, owner): + if value < 0 or value > MAX_BUILD_MEMORY_MB: + fail("{}: build_memory_mb must be between 0 and {}".format( + owner, + MAX_BUILD_MEMORY_MB, + )) diff --git a/uv/private/pep517_whl/memory_monitor.py b/uv/private/pep517_whl/memory_monitor.py new file mode 100644 index 000000000..06736f845 --- /dev/null +++ b/uv/private/pep517_whl/memory_monitor.py @@ -0,0 +1,168 @@ +"""Run a wheel build while reporting its aggregate process-tree memory.""" + +import os +import signal +import subprocess +import sys +import time + +_MIB = 1024 * 1024 +_REPORT_STEP_BYTES = 256 * _MIB +_SAMPLE_INTERVAL_SECONDS = 0.25 + + +def _exit_on_sigterm(signum, _frame): + raise SystemExit(128 + signum) + + +_children_unavailable_warned = False + + +def _process_tree_rss_bytes(root_pid, page_size, proc_root="/proc"): + """Return aggregate RSS for root_pid and its descendants on Linux.""" + global _children_unavailable_warned + # statm reports resident memory in pages: + # https://man7.org/linux/man-pages/man5/proc_pid_statm.5.html + # + # Each task's children file lists only immediate children, so recurse: + # https://docs.kernel.org/filesystems/proc.html#proc-pid-task-tid-children-information-about-task-children + pending = [root_pid] + seen = set() + total = 0 + while pending: + pid = pending.pop() + if pid in seen: + continue + seen.add(pid) + process_dir = os.path.join(proc_root, str(pid)) + + try: + with open(os.path.join(process_dir, "statm")) as statm: + total += int(statm.read().split()[1]) * page_size + task_ids = os.listdir(os.path.join(process_dir, "task")) + except ( + FileNotFoundError, + PermissionError, + ProcessLookupError, + ValueError, + IndexError, + ): + continue + + any_children_file_found = False + for task_id in task_ids: + try: + with open( + os.path.join(process_dir, "task", task_id, "children") + ) as children: + pending.extend(int(child) for child in children.read().split()) + any_children_file_found = True + except FileNotFoundError: + pass # children file absent; may indicate missing CONFIG_PROC_CHILDREN + except (PermissionError, ProcessLookupError, ValueError): + any_children_file_found = True # file exists; transient access error + continue + + if task_ids and not any_children_file_found and not _children_unavailable_warned: + _children_unavailable_warned = True + print( + "rules_py: /proc/{pid}/task/{tid}/children is unavailable; " + "memory sampling covers only the root process " + "(kernel may lack CONFIG_PROC_CHILDREN).", + file=sys.stderr, + flush=True, + ) + + return total + + +def _report_memory(wheel, state, current, peak): + if peak is None: + details = "unavailable" + elif current is None: + details = "sampled aggregate peak={:.1f} MiB".format(peak / _MIB) + else: + details = "sampled aggregate current={:.1f} MiB, peak={:.1f} MiB".format( + current / _MIB, + peak / _MIB, + ) + print( + "rules_py wheel build memory for {} {}: {}".format( + wheel, + state, + details, + ), + file=sys.stderr, + flush=True, + ) + + +def run_with_memory_profile(cmd, *, cwd, env, stdout, wheel, monitor): + """Run cmd, optionally reporting process-tree memory.""" + _sp_kwargs = {"cwd": cwd, "env": env, "stdout": stdout, "stderr": subprocess.STDOUT} + + if not monitor: + subprocess.run(cmd, check=True, **_sp_kwargs) + return + + can_sample_proc = sys.platform.startswith("linux") and os.path.isdir("/proc") + try: + page_size = os.sysconf("SC_PAGE_SIZE") if can_sample_proc else None + except (OSError, ValueError): + can_sample_proc = False + page_size = None + + if not can_sample_proc: + try: + subprocess.run(cmd, check=True, **_sp_kwargs) + except subprocess.CalledProcessError: + _report_memory(wheel, "finished", None, None) + raise + _report_memory(wheel, "finished", None, None) + return + + previous_sigterm_handler = signal.signal(signal.SIGTERM, _exit_on_sigterm) + process = None + + try: + process = subprocess.Popen( + cmd, + start_new_session=True, + **_sp_kwargs, + ) + peak = 0 + next_report = _REPORT_STEP_BYTES + + while True: + if can_sample_proc: + try: + current = _process_tree_rss_bytes(process.pid, page_size) + except (OSError, ValueError): + can_sample_proc = False + peak = None + else: + peak = max(peak, current) + if peak >= next_report: + _report_memory(wheel, "running", current, peak) + next_report = ( + peak // _REPORT_STEP_BYTES + 1 + ) * _REPORT_STEP_BYTES + + returncode = process.poll() + if returncode is not None: + break + time.sleep(_SAMPLE_INTERVAL_SECONDS) + except BaseException: + if process is not None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + raise + finally: + signal.signal(signal.SIGTERM, previous_sigterm_handler) + + _report_memory(wheel, "finished", None, peak or None) + if returncode: + raise subprocess.CalledProcessError(returncode, cmd) diff --git a/uv/private/pep517_whl/memory_monitor_test.py b/uv/private/pep517_whl/memory_monitor_test.py new file mode 100644 index 000000000..d885b7b8b --- /dev/null +++ b/uv/private/pep517_whl/memory_monitor_test.py @@ -0,0 +1,219 @@ +import contextlib +import io +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + +from uv.private.pep517_whl import memory_monitor + + +class ProcessTreeRssTest(unittest.TestCase): + def test_sums_descendants_spawned_by_any_task(self): + with tempfile.TemporaryDirectory() as directory: + proc_root = Path(directory) + self._write_process(proc_root, 10, 2, {10: [20], 11: [30]}) + self._write_process(proc_root, 20, 3, {20: []}) + self._write_process(proc_root, 30, 5, {30: []}) + + self.assertEqual( + 10 * 4096, + memory_monitor._process_tree_rss_bytes(10, 4096, str(proc_root)), + ) + + def test_missing_children_file_warns_and_covers_only_root(self): + """When children files are absent (CONFIG_PROC_CHILDREN missing), the walk + stops at the root PID and emits a one-time warning to stderr.""" + stderr = io.StringIO() + with tempfile.TemporaryDirectory() as directory: + proc_root = Path(directory) + + process_dir = proc_root / "10" + process_dir.mkdir() + (process_dir / "statm").write_text("100 5 0 0 0 0 0\n") + (process_dir / "task" / "10").mkdir(parents=True) + child_dir = proc_root / "20" + child_dir.mkdir() + (child_dir / "statm").write_text("100 3 0 0 0 0 0\n") + + with ( + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor, "_children_unavailable_warned", False), + ): + result = memory_monitor._process_tree_rss_bytes(10, 4096, str(proc_root)) + + self.assertEqual(5 * 4096, result) # only root PID; process 20 unreachable + self.assertIn("CONFIG_PROC_CHILDREN", stderr.getvalue()) + + def test_success_without_monitoring_uses_run(self): + with tempfile.TemporaryFile() as stdout: + memory_monitor.run_with_memory_profile( + [sys.executable, "-c", "pass"], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=False, + ) + + def test_failed_child_reports_memory_before_raising(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor, "_REPORT_STEP_BYTES", 1), + ): + with self.assertRaisesRegex(subprocess.CalledProcessError, "exit status 7"): + memory_monitor.run_with_memory_profile( + [ + sys.executable, + "-c", + "import time; data = bytearray(16 * 1024 * 1024); time.sleep(.5); raise SystemExit(7)", + ], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + report = stderr.getvalue() + if sys.platform.startswith("linux") and os.path.isdir("/proc"): + self.assertRegex( + report, + r"wheel build memory for test-wheel[.]tar[.]gz running: sampled aggregate current=[1-9][0-9.]* MiB, peak=[1-9][0-9.]* MiB", + ) + self.assertRegex( + report, + r"wheel build memory for test-wheel[.]tar[.]gz finished: sampled aggregate peak=[1-9][0-9.]* MiB", + ) + else: + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + report, + ) + + def test_sampler_failure_reports_unavailable(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor.sys, "platform", "linux"), + mock.patch.object(memory_monitor.os.path, "isdir", return_value=True), + mock.patch.object( + memory_monitor, + "_process_tree_rss_bytes", + side_effect=OSError("procfs unavailable"), + ), + ): + memory_monitor.run_with_memory_profile( + [sys.executable, "-c", "import time; time.sleep(.1)"], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + stderr.getvalue(), + ) + + def test_unsupported_platform_uses_run(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor.sys, "platform", "win32"), + mock.patch.object(memory_monitor.subprocess, "run") as run, + ): + memory_monitor.run_with_memory_profile( + ["command"], + cwd=None, + env={}, + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + run.assert_called_once() + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + stderr.getvalue(), + ) + + @unittest.skipUnless( + sys.platform.startswith("linux") and os.path.isdir("/proc"), + "requires Linux procfs", + ) + def test_exception_kills_and_waits_for_process_tree(self): + for interrupt_signal, expected_exception in ( + (signal.SIGINT, KeyboardInterrupt), + (signal.SIGTERM, SystemExit), + ): + with ( + self.subTest(signal=interrupt_signal), + tempfile.TemporaryDirectory() as directory, + tempfile.TemporaryFile() as stdout, + ): + pids_file = Path(directory) / "pids" + with self.assertRaises(expected_exception) as raised: + memory_monitor.run_with_memory_profile( + [ + sys.executable, + "-c", + "import os, pathlib, signal, subprocess, sys, time; " + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)']); " + "pathlib.Path(sys.argv[1]).write_text(f'{os.getpid()} {child.pid}'); " + "os.kill(os.getppid(), int(sys.argv[2])); " + "time.sleep(60)", + str(pids_file), + str(interrupt_signal), + ], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + if interrupt_signal == signal.SIGTERM: + self.assertEqual(128 + signal.SIGTERM, raised.exception.code) + + frontend_pid, child_pid = [ + int(pid) for pid in pids_file.read_text().split() + ] + for pid in (frontend_pid, child_pid): + deadline = time.monotonic() + 5 + while True: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + if time.monotonic() >= deadline: + self.fail( + "process {} survived interruption cleanup".format(pid) + ) + time.sleep(0.01) + + @staticmethod + def _write_process(proc_root, pid, rss_pages, task_children): + process_dir = proc_root / str(pid) + process_dir.mkdir() + (process_dir / "statm").write_text("100 {} 0 0 0 0 0\n".format(rss_pages)) + for task_id, children in task_children.items(): + task_dir = process_dir / "task" / str(task_id) + task_dir.mkdir(parents=True) + (task_dir / "children").write_text( + " ".join(str(child) for child in children) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv/private/pep517_whl/rule.bzl b/uv/private/pep517_whl/rule.bzl index 6677d1f9c..c83c4d292 100644 --- a/uv/private/pep517_whl/rule.bzl +++ b/uv/private/pep517_whl/rule.bzl @@ -5,8 +5,10 @@ Uses `python -m build` (the pypa/build frontend) which delegates to whatever build backend the sdist declares in its `[build-system]` table. """ +load("@bazel_lib//lib:resource_sets.bzl", "resource_set_for") load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") load("//py/private/toolchain:types.bzl", "NATIVE_BUILD_TOOLCHAIN", "PY_TOOLCHAIN") +load(":build_memory.bzl", "validate_build_memory_mb") def _common_env(ctx): return { @@ -28,6 +30,13 @@ def _patch_args_and_inputs(ctx): patch_inputs.append(f) return patch_args, patch_inputs +def _memory_args(ctx): + return ["--monitor-memory"] if ctx.attr.build_memory_mb else [] + +def _build_resource_set(ctx): + validate_build_memory_mb(ctx.attr.build_memory_mb, ctx.label) + return resource_set_for(mem_mb = ctx.attr.build_memory_mb) + def _collect_toolchain_inputs_and_vars(ctx): """Gather files + Make-variable substitutions from `ctx.attr.toolchains`. @@ -96,7 +105,7 @@ def _cc_toolchain_inputs_and_compiler(ctx): def _pep517_whl(ctx): archive = ctx.file.src - wheel_dir = ctx.actions.declare_directory("whl") + wheel_dir = ctx.actions.declare_directory(ctx.label.name) patch_args, patch_inputs = _patch_args_and_inputs(ctx) # The build tool is a py_binary wrapping build_helper.py. Using it as @@ -109,7 +118,7 @@ def _pep517_whl(ctx): progress_message = "Source compiling {} to a whl".format(archive.basename), executable = ctx.executable.tool, toolchain = None, - arguments = ctx.attr.args + patch_args + [ + arguments = ctx.attr.args + patch_args + _memory_args(ctx) + [ archive.path, wheel_dir.path, ], @@ -118,13 +127,14 @@ def _pep517_whl(ctx): outputs = [wheel_dir], env = _common_env(ctx), exec_group = "target", + resource_set = _build_resource_set(ctx), ) return [DefaultInfo(files = depset([wheel_dir]))] def _pep517_native_whl(ctx): archive = ctx.file.src - wheel_dir = ctx.actions.declare_directory("whl") + wheel_dir = ctx.actions.declare_directory(ctx.label.name) patch_args, patch_inputs = _patch_args_and_inputs(ctx) env = _common_env(ctx) @@ -146,7 +156,7 @@ def _pep517_native_whl(ctx): progress_message = "Native source compiling {} to a whl".format(archive.basename), executable = ctx.executable.tool, toolchain = None, - arguments = ctx.attr.args + patch_args + [ + arguments = ctx.attr.args + patch_args + _memory_args(ctx) + [ archive.path, wheel_dir.path, ], @@ -158,6 +168,7 @@ def _pep517_native_whl(ctx): outputs = [wheel_dir], env = env, exec_group = "target", + resource_set = _build_resource_set(ctx), ) return [DefaultInfo(files = depset([wheel_dir]))] @@ -175,6 +186,12 @@ _PATCH_ATTRS = { } _pep517_whl_attrs = { + "build_memory_mb": attr.int( + default = 0, + doc = "Estimated peak memory in MB for local wheel builds, from 0 to " + + "32768. Bazel rounds this up to the next resource class supported " + + "by bazel_lib. Zero uses Bazel's default estimate.", + ), "src": attr.label(allow_single_file = True), "tool": attr.label(executable = True, cfg = "exec"), "version": attr.string(), diff --git a/uv/private/pep517_whl/test.bzl b/uv/private/pep517_whl/test.bzl index 9f9b758cb..510a6a900 100644 --- a/uv/private/pep517_whl/test.bzl +++ b/uv/private/pep517_whl/test.bzl @@ -17,6 +17,26 @@ _JDK_ENV_KEYS = ["JAVA_HOME", "JAVA", "JAR"] _REQUIRED_ENV_KEYS = _CC_ENV_KEYS + _JDK_ENV_KEYS +def _analysis_failure_test_impl(ctx): + env = analysistest.begin(ctx) + asserts.expect_failure(env, ctx.attr.expected_error) + return analysistest.end(env) + +build_memory_failure_test = analysistest.make( + _analysis_failure_test_impl, + attrs = {"expected_error": attr.string()}, + expect_failure = True, +) + +def _build_memory_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + actions = [action for action in target.actions if action.mnemonic == "PySdistBuild"] + asserts.equals(env, 1, len(actions)) + return analysistest.end(env) + +build_memory_test = analysistest.make(_build_memory_test_impl) + def _toolchain_env_test_impl(ctx): env = analysistest.begin(ctx) target = analysistest.target_under_test(env) diff --git a/uv/private/sdist_build/repository.bzl b/uv/private/sdist_build/repository.bzl index a64df625f..e57ffba8e 100644 --- a/uv/private/sdist_build/repository.bzl +++ b/uv/private/sdist_build/repository.bzl @@ -203,6 +203,9 @@ def _sdist_build_impl(repository_ctx): # Merge explicit deps with auto-discovered deps all_deps = [str(d) for d in repository_ctx.attr.deps] + extra_dep_labels + build_tool_deps = [ + "@aspect_rules_py//uv/private/pep517_whl:memory_monitor", + ] + all_deps pre_build_patches = repository_ctx.attr.pre_build_patches patch_attrs = "" @@ -250,6 +253,10 @@ def _sdist_build_impl(repository_ctx): env = "\n".join([" \"{}\": \"{}\",".format(k, v) for k, v in sorted(env.items())]), ) + memory_attr = "" + if repository_ctx.attr.build_memory_mb: + memory_attr = "\n build_memory_mb = {},".format(repository_ctx.attr.build_memory_mb) + repository_ctx.file("BUILD.bazel", content = """ load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "{rule}") load("@aspect_rules_py//py:defs.bzl", "py_binary") @@ -266,7 +273,7 @@ py_binary( src = "{src}", tool = ":build_tool", version = "{version}", - args = [],{patch_attrs}{toolchain_attrs} + args = [],{memory_attr}{patch_attrs}{toolchain_attrs} visibility = ["//visibility:public"], ) @@ -276,9 +283,10 @@ exports_files( ) """.format( src = repository_ctx.attr.src, - deps = repr(all_deps), + deps = repr(build_tool_deps), rule = "pep517_native_whl" if is_native else "pep517_whl", version = repository_ctx.attr.version, + memory_attr = memory_attr, patch_attrs = patch_attrs, toolchain_attrs = toolchain_attrs, )) @@ -303,6 +311,10 @@ sdist_build = repository_rule( "two arguments. See //uv/private/sdist_configure:defs.bzl.", ), "version": attr.string(), + "build_memory_mb": attr.int( + default = 0, + doc = "Estimated peak memory in MB for local wheel builds, from 0 to 32768. Set via uv.override_package(build_memory_mb = ...).", + ), "pre_build_patches": attr.label_list(default = []), "pre_build_patch_strip": attr.int(default = 0), "extra_toolchains": attr.string_list( diff --git a/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel b/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel index d8ca27afe..54d448b05 100644 --- a/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel +++ b/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel @@ -6,7 +6,7 @@ py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@@+uv+project__aspect_rules_py//:build", "@@+uv+project__aspect_rules_py//:setuptools", "@@+uv+project__aspect_rules_py//:maturin", "@whl_install__aspect_rules_py__setuptools__80_9_0//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@+uv+project__aspect_rules_py//:build", "@@+uv+project__aspect_rules_py//:setuptools", "@@+uv+project__aspect_rules_py//:maturin", "@whl_install__aspect_rules_py__setuptools__80_9_0//:install"], ) pep517_whl( From 7b2e5e4b57ec0378c5f64be34c44cf55e1555299 Mon Sep 17 00:00:00 2001 From: xangcastle Date: Mon, 22 Jun 2026 21:50:26 -0600 Subject: [PATCH 2/6] xangcastle/uv-1-wheel-build-memory --- MODULE.bazel | 2 +- docs/uv-patching.md | 22 ++ e2e/BUILD.bazel | 9 +- .../uv-sdist-jdk-build/setup.MODULE.bazel | 6 +- e2e/cases/uv-sdist-mpicc/BUILD.bazel | 5 +- e2e/cases/uv-sdist-mpicc/__test__.py | 8 + ...uild.uv_sdist_jdk_build.jpype1.BUILD.bazel | 3 +- uv/private/extension/BUILD.bazel | 1 + uv/private/extension/defs.bzl | 24 +- uv/private/pep517_whl/BUILD.bazel | 58 ++++- uv/private/pep517_whl/build_helper.py | 14 +- uv/private/pep517_whl/build_memory.bzl | 12 + uv/private/pep517_whl/memory_monitor.py | 168 ++++++++++++++ uv/private/pep517_whl/memory_monitor_test.py | 219 ++++++++++++++++++ uv/private/pep517_whl/rule.bzl | 30 ++- uv/private/pep517_whl/test.bzl | 20 ++ uv/private/sdist_build/repository.bzl | 16 +- .../sdist_build.bravado_core.BUILD.bazel | 2 +- 18 files changed, 590 insertions(+), 29 deletions(-) create mode 100644 uv/private/pep517_whl/build_memory.bzl create mode 100644 uv/private/pep517_whl/memory_monitor.py create mode 100644 uv/private/pep517_whl/memory_monitor_test.py diff --git a/MODULE.bazel b/MODULE.bazel index 2fcc25c65..75ebf0260 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ module( ) bazel_dep(name = "bazel_features", version = "1.38.0") -bazel_dep(name = "bazel_lib", version = "3.0.0") +bazel_dep(name = "bazel_lib", version = "3.2.0") bazel_dep(name = "bazel_skylib", version = "1.4.2") bazel_dep(name = "gawk", version = "5.3.2.bcr.7") bazel_dep(name = "platforms", version = "1.0.0") diff --git a/docs/uv-patching.md b/docs/uv-patching.md index 0f3fb4dd6..2732eaad5 100644 --- a/docs/uv-patching.md +++ b/docs/uv-patching.md @@ -83,6 +83,28 @@ Pre-build patches are applied to the extracted source tree after archive extract - Removing problematic native build dependencies - Patching source code that affects the build output +### Allocating wheel build memory + +Set `build_memory_mb` to the expected peak memory of a local sdist build so +Bazel can limit concurrent wheel builds: + +```starlark +uv.override_package( + lock = "//:uv.lock", + name = "native-package", + build_memory_mb = 6144, +) +``` + +Bazel rounds the estimate up to one of its supported resource classes. On +Linux, setting a nonzero estimate also enables aggregate resident-memory +sampling for the build process and its descendants. Estimates above 32768 MiB +are rejected because bazel-lib 3.2 would otherwise clamp them to 32768 MiB and +silently under-reserve the action. A measurement appears whenever the sampled +peak crosses another 256 MiB threshold and once when the build finishes. If +procfs is unavailable or cannot be read, the final measurement is reported as +unavailable. + ### Adding extra dependencies or data Some packages have implicit runtime dependencies that aren't declared in their metadata: diff --git a/e2e/BUILD.bazel b/e2e/BUILD.bazel index 28d2a93a9..f88936b6e 100644 --- a/e2e/BUILD.bazel +++ b/e2e/BUILD.bazel @@ -110,13 +110,14 @@ write_source_files( # shows up as the per-wheel dicts collapsing into one merged list. "snapshots/abi3_compat.whl_install.cffi.BUILD.bazel": "@whl_install__abi3_compat__cffi__2_0_0//:BUILD.bazel", - # @sdist_build for jpype1 with uv.override_package toolchains/env + # @sdist_build for jpype1 with uv.override_package build resources, + # toolchains, and env # ---------------------------------------------------------------- # Covers the override_package -> repo-rule -> BUILD-template - # plumbing for the augment-on-defaults `toolchains = [...]` / + # plumbing for `build_memory_mb`, `toolchains = [...]`, and # `env = {...}`. The override layers a JDK runtime on top of the - # default CC toolchain; the snapshot pins both toolchain - # references and both env families (CC + JAVA_HOME/JAVA/JAR). + # default CC toolchain; the snapshot pins the memory declaration, + # toolchain references, and both env families (CC + JAVA_HOME/JAVA/JAR). # Pairs with the analysis test at # //uv/private/pep517_whl:toolchain_env_test which asserts the # rule expands env keys correctly given an explicit fixture. diff --git a/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel b/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel index 83f105fb0..e696558bc 100644 --- a/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel +++ b/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel @@ -23,11 +23,11 @@ uv.unstable_annotate_packages( lock = "//cases/uv-sdist-jdk-build:uv.lock", ) -# Layer a JDK runtime toolchain into the build env. `toolchains` and -# `env` here are additive — these entries are appended to whatever -# sdist_build's BUILD template emits by default. +# Reserve local build memory and layer a JDK runtime toolchain into the build +# env. `toolchains` and `env` are additive to sdist_build's defaults. uv.override_package( name = "jpype1", + build_memory_mb = 2048, env = { "JAR": "$(JAVABASE)/bin/jar", "JAVA": "$(JAVA)", diff --git a/e2e/cases/uv-sdist-mpicc/BUILD.bazel b/e2e/cases/uv-sdist-mpicc/BUILD.bazel index 46a439565..8224e48c5 100644 --- a/e2e/cases/uv-sdist-mpicc/BUILD.bazel +++ b/e2e/cases/uv-sdist-mpicc/BUILD.bazel @@ -9,7 +9,10 @@ load("@aspect_rules_py//py:defs.bzl", "py_test") py_test( name = "test", srcs = ["__test__.py"], - data = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], + data = [ + "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", + "@aspect_rules_py//uv/private/pep517_whl:memory_monitor", + ], dep_group = "uv_sdist_mpicc", main = "__test__.py", python_version = "3.11", diff --git a/e2e/cases/uv-sdist-mpicc/__test__.py b/e2e/cases/uv-sdist-mpicc/__test__.py index c10027653..192664d50 100644 --- a/e2e/cases/uv-sdist-mpicc/__test__.py +++ b/e2e/cases/uv-sdist-mpicc/__test__.py @@ -76,10 +76,18 @@ def make_sdist(workdir): def run_helper(helper, sdist, outdir, path_entries, expect): + from pathlib import Path as _Path + + helper_path = _Path(helper) + assert helper_path.parts[-4:] == ("uv", "private", "pep517_whl", "build_helper.py"), ( + "Unexpected helper path structure: {}".format(helper) + ) + rules_py_root = str(helper_path.parents[3]) env = { "MPICC_TEST_EXPECT": expect, "PATH": os.pathsep.join(path_entries), "HOME": os.environ.get("TEST_TMPDIR", "/tmp"), + "PYTHONPATH": rules_py_root, } cc = shutil.which("cc") or "/usr/bin/cc" if os.path.exists(cc): diff --git a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel index daa114fdc..4e25a55c9 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel @@ -6,7 +6,7 @@ py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:scikit_build_core", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:setuptools", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:wheel", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:build", "@whl_install__uv_sdist_jdk_build__scikit_build_core__0_12_2//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:scikit_build_core", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:setuptools", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:wheel", "@@aspect_rules_py++uv+project__uv_sdist_jdk_build//:build", "@whl_install__uv_sdist_jdk_build__scikit_build_core__0_12_2//:install"], ) pep517_native_whl( @@ -15,6 +15,7 @@ pep517_native_whl( tool = ":build_tool", version = "1.7.1", args = [], + build_memory_mb = 2048, toolchains = [ "@bazel_tools//tools/cpp:current_cc_toolchain", "@@bazel_tools//tools/jdk:current_java_runtime", diff --git a/uv/private/extension/BUILD.bazel b/uv/private/extension/BUILD.bazel index 2ac723012..768403783 100644 --- a/uv/private/extension/BUILD.bazel +++ b/uv/private/extension/BUILD.bazel @@ -22,6 +22,7 @@ bzl_library( "//uv/private:normalize_version", "//uv/private/constraints:repository", "//uv/private/git_archive:repository", + "//uv/private/pep517_whl:build_memory", "//uv/private/pprint:defs", "//uv/private/sdist_build:repository", "//uv/private/sdist_configure:defs", diff --git a/uv/private/extension/defs.bzl b/uv/private/extension/defs.bzl index 38584211e..f4fcb2cfc 100644 --- a/uv/private/extension/defs.bzl +++ b/uv/private/extension/defs.bzl @@ -61,6 +61,7 @@ load("//uv/private:normalize_version.bzl", "normalize_version") load("//uv/private:parse_whl_name.bzl", "parse_whl_name") load("//uv/private/constraints:repository.bzl", "configurations_hub") load("//uv/private/git_archive:repository.bzl", "git_archive") +load("//uv/private/pep517_whl:build_memory.bzl", "validate_build_memory_mb") load("//uv/private/pprint:defs.bzl", "pprint") load("//uv/private/sdist_build:repository.bzl", "sdist_build") load("//uv/private/sdist_configure:defs.bzl", "DEFAULT_CONFIGURE_SCRIPT") @@ -247,6 +248,10 @@ def _parse_projects(module_ctx, hub_specs): )) has_target = override.target != None + validate_build_memory_mb( + override.build_memory_mb, + "uv.override_package() for '{}'".format(override.name), + ) has_modifications = ( override.pre_build_patches or override.post_install_patches or @@ -254,14 +259,15 @@ def _parse_projects(module_ctx, hub_specs): override.extra_data or override.toolchains or override.env or - override.resource_set != "default" + override.resource_set != "default" or + override.build_memory_mb ) if has_target and has_modifications: fail("uv.override_package() for '{}': `target` is mutually exclusive with modification attributes. Use `target` for full replacement OR build, patch, and data attributes for modifications, not both.".format(override.name)) if not has_target and not has_modifications: - fail("uv.override_package() for '{}': must specify either `target` for full replacement or at least one modification attribute (pre_build_patches, post_install_patches, extra_deps, extra_data, toolchains, env, resource_set).".format(override.name)) + fail("uv.override_package() for '{}': must specify either `target` for full replacement or at least one modification attribute (pre_build_patches, post_install_patches, extra_deps, extra_data, toolchains, env, resource_set, build_memory_mb).".format(override.name)) package_overrides[override_key] = override @@ -413,10 +419,12 @@ def _parse_projects(module_ctx, hub_specs): extra_toolchains = [] extra_env = {} resource_set = "default" + build_memory_mb = 0 if pkg_override: extra_toolchains = [str(t) for t in pkg_override.toolchains] extra_env = pkg_override.env resource_set = pkg_override.resource_set + build_memory_mb = pkg_override.build_memory_mb sbuild_specs[sbuild_id] = struct( src = sdist, @@ -430,6 +438,7 @@ def _parse_projects(module_ctx, hub_specs): extra_toolchains = extra_toolchains, extra_env = extra_env, resource_set = resource_set, + build_memory_mb = build_memory_mb, ) has_sbuild = True @@ -659,6 +668,8 @@ def _uv_impl(module_ctx): sbuild_kwargs["extra_env"] = sbuild_cfg.extra_env if sbuild_cfg.resource_set != "default": sbuild_kwargs["resource_set"] = sbuild_cfg.resource_set + if sbuild_cfg.build_memory_mb: + sbuild_kwargs["build_memory_mb"] = sbuild_cfg.build_memory_mb sdist_build(**sbuild_kwargs) for install_id, install_cfg in cfg.install_cfgs.items(): @@ -770,12 +781,16 @@ _override_package_tag = tag_class( "bucket.", ), - # Per-package toolchain plumbing for native sdist builds. Both - # attributes AUGMENT the defaults baked into sdist_build's + # Per-package resource and toolchain settings for sdist builds. + # toolchains and env AUGMENT the defaults baked into sdist_build's # generated `pep517_native_whl(...)` call (the CC toolchain + # CC/CXX/AR/LD/STRIP env) — they don't replace them. Use these # to layer extra toolchains (Java runtime, Rust, …) and extra # env vars on top of the defaults. + "build_memory_mb": attr.int( + default = 0, + doc = "Estimated peak memory in MB for this package's local wheel build, from 0 to 32768. Bazel rounds up to a supported resource class; zero uses its default estimate.", + ), "toolchains": attr.label_list( default = [], doc = "Extra toolchain targets appended to the generated pep517_native_whl(...) call's `toolchains` list. Each target's TemplateVariableInfo make-variables become available for $(VAR) expansion in `env`.", @@ -784,7 +799,6 @@ _override_package_tag = tag_class( default = {}, doc = "Extra environment variables merged into the build action's `env` dict. Values may reference $(VAR) make-variables sourced from the default CC toolchain or any extra `toolchains` listed above.", ), - # Pre-build patches: applied to extracted sdist source before wheel build. "pre_build_patches": attr.label_list( default = [], diff --git a/uv/private/pep517_whl/BUILD.bazel b/uv/private/pep517_whl/BUILD.bazel index 5b52d993a..f5e84cb1e 100644 --- a/uv/private/pep517_whl/BUILD.bazel +++ b/uv/private/pep517_whl/BUILD.bazel @@ -1,8 +1,14 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load(":rule.bzl", "pep517_native_whl") -load(":test.bzl", "pep517_native_whl_toolchain_env_test") +load("//py:defs.bzl", "py_library", "py_test") +load(":rule.bzl", "pep517_native_whl", "pep517_whl") +load( + ":test.bzl", + "build_memory_failure_test", + "build_memory_test", + "pep517_native_whl_toolchain_env_test", +) package(default_visibility = [ "//uv/private:__subpackages__", @@ -17,11 +23,25 @@ bzl_library( name = "rule", srcs = ["rule.bzl"], deps = [ + ":build_memory", "//py/private/toolchain:types", "@bazel_lib//lib:resource_sets", ], ) +py_library( + name = "memory_monitor", + srcs = ["memory_monitor.py"], + visibility = ["//visibility:public"], +) + +py_test( + name = "memory_monitor_test", + srcs = ["memory_monitor_test.py"], + main = "memory_monitor_test.py", + deps = [":memory_monitor"], +) + bzl_library( name = "test", srcs = ["test.bzl"], @@ -62,6 +82,35 @@ sh_binary( tags = ["manual"], ) +pep517_whl( + name = "__excessive_memory_fixture", + src = ":__stub_sdist", + build_memory_mb = 32769, + tags = ["manual"], + tool = ":__stub_tool", + version = "0.0.1", +) + +build_memory_failure_test( + name = "build_memory_limit_test", + expected_error = "build_memory_mb must be between 0 and 32768", + target_under_test = ":__excessive_memory_fixture", +) + +pep517_whl( + name = "__maximum_memory_fixture", + src = ":__stub_sdist", + build_memory_mb = 32768, + tags = ["manual"], + tool = ":__stub_tool", + version = "0.0.1", +) + +build_memory_test( + name = "build_memory_maximum_test", + target_under_test = ":__maximum_memory_fixture", +) + # Fixture replicates the toolchains + env defaults emitted by sdist_build's # BUILD template (see uv/private/sdist_build/repository.bzl). The analysis # test below asserts these expand to non-empty toolchain paths. @@ -91,3 +140,8 @@ pep517_native_whl_toolchain_env_test( name = "toolchain_env_test", target_under_test = ":__toolchain_env_fixture", ) + +bzl_library( + name = "build_memory", + srcs = ["build_memory.bzl"], +) diff --git a/uv/private/pep517_whl/build_helper.py b/uv/private/pep517_whl/build_helper.py index 10774c6c3..8a5e9d3dc 100644 --- a/uv/private/pep517_whl/build_helper.py +++ b/uv/private/pep517_whl/build_helper.py @@ -13,9 +13,11 @@ import shutil import sys from os import chmod, defpath, listdir, makedirs, path, pathsep -from subprocess import CalledProcessError, check_call, check_output, STDOUT, run +from subprocess import CalledProcessError, check_call, check_output, STDOUT from tempfile import TemporaryFile +from uv.private.pep517_whl.memory_monitor import run_with_memory_profile + try: import tomllib except ModuleNotFoundError: @@ -187,6 +189,7 @@ def _legacy_metadata_conflicts_with_pyproject(worktree): PARSER = ArgumentParser() PARSER.add_argument("srcarchive") PARSER.add_argument("outdir") +PARSER.add_argument("--monitor-memory", action="store_true") PARSER.add_argument("--validate-anyarch", action="store_true") PARSER.add_argument("--patch-strip", type=int, default=0, help="Strip count for patch (-p)") PARSER.add_argument("--patch", action="append", default=[], dest="patches", help="Patch file to apply (repeatable)") @@ -279,7 +282,14 @@ def _legacy_metadata_conflicts_with_pyproject(worktree): with TemporaryFile(mode="w+") as build_log: try: - run(cmd, cwd=t, env=build_env, stdout=build_log, stderr=STDOUT, check=True) + run_with_memory_profile( + cmd, + cwd=t, + env=build_env, + stdout=build_log, + wheel=path.basename(opts.srcarchive), + monitor=opts.monitor_memory, + ) except CalledProcessError: build_log.seek(0) output = build_log.read() diff --git a/uv/private/pep517_whl/build_memory.bzl b/uv/private/pep517_whl/build_memory.bzl new file mode 100644 index 000000000..17297ede9 --- /dev/null +++ b/uv/private/pep517_whl/build_memory.bzl @@ -0,0 +1,12 @@ +"""Validation shared by wheel build configuration surfaces.""" + +# bazel-lib 3.2 clamps larger resource requests to 32 GiB: +# https://github.com/aspect-build/bazel-lib/blob/v3.2.0/lib/private/resource_sets.bzl#L2224-L2227 +MAX_BUILD_MEMORY_MB = 32768 + +def validate_build_memory_mb(value, owner): + if value < 0 or value > MAX_BUILD_MEMORY_MB: + fail("{}: build_memory_mb must be between 0 and {}".format( + owner, + MAX_BUILD_MEMORY_MB, + )) diff --git a/uv/private/pep517_whl/memory_monitor.py b/uv/private/pep517_whl/memory_monitor.py new file mode 100644 index 000000000..06736f845 --- /dev/null +++ b/uv/private/pep517_whl/memory_monitor.py @@ -0,0 +1,168 @@ +"""Run a wheel build while reporting its aggregate process-tree memory.""" + +import os +import signal +import subprocess +import sys +import time + +_MIB = 1024 * 1024 +_REPORT_STEP_BYTES = 256 * _MIB +_SAMPLE_INTERVAL_SECONDS = 0.25 + + +def _exit_on_sigterm(signum, _frame): + raise SystemExit(128 + signum) + + +_children_unavailable_warned = False + + +def _process_tree_rss_bytes(root_pid, page_size, proc_root="/proc"): + """Return aggregate RSS for root_pid and its descendants on Linux.""" + global _children_unavailable_warned + # statm reports resident memory in pages: + # https://man7.org/linux/man-pages/man5/proc_pid_statm.5.html + # + # Each task's children file lists only immediate children, so recurse: + # https://docs.kernel.org/filesystems/proc.html#proc-pid-task-tid-children-information-about-task-children + pending = [root_pid] + seen = set() + total = 0 + while pending: + pid = pending.pop() + if pid in seen: + continue + seen.add(pid) + process_dir = os.path.join(proc_root, str(pid)) + + try: + with open(os.path.join(process_dir, "statm")) as statm: + total += int(statm.read().split()[1]) * page_size + task_ids = os.listdir(os.path.join(process_dir, "task")) + except ( + FileNotFoundError, + PermissionError, + ProcessLookupError, + ValueError, + IndexError, + ): + continue + + any_children_file_found = False + for task_id in task_ids: + try: + with open( + os.path.join(process_dir, "task", task_id, "children") + ) as children: + pending.extend(int(child) for child in children.read().split()) + any_children_file_found = True + except FileNotFoundError: + pass # children file absent; may indicate missing CONFIG_PROC_CHILDREN + except (PermissionError, ProcessLookupError, ValueError): + any_children_file_found = True # file exists; transient access error + continue + + if task_ids and not any_children_file_found and not _children_unavailable_warned: + _children_unavailable_warned = True + print( + "rules_py: /proc/{pid}/task/{tid}/children is unavailable; " + "memory sampling covers only the root process " + "(kernel may lack CONFIG_PROC_CHILDREN).", + file=sys.stderr, + flush=True, + ) + + return total + + +def _report_memory(wheel, state, current, peak): + if peak is None: + details = "unavailable" + elif current is None: + details = "sampled aggregate peak={:.1f} MiB".format(peak / _MIB) + else: + details = "sampled aggregate current={:.1f} MiB, peak={:.1f} MiB".format( + current / _MIB, + peak / _MIB, + ) + print( + "rules_py wheel build memory for {} {}: {}".format( + wheel, + state, + details, + ), + file=sys.stderr, + flush=True, + ) + + +def run_with_memory_profile(cmd, *, cwd, env, stdout, wheel, monitor): + """Run cmd, optionally reporting process-tree memory.""" + _sp_kwargs = {"cwd": cwd, "env": env, "stdout": stdout, "stderr": subprocess.STDOUT} + + if not monitor: + subprocess.run(cmd, check=True, **_sp_kwargs) + return + + can_sample_proc = sys.platform.startswith("linux") and os.path.isdir("/proc") + try: + page_size = os.sysconf("SC_PAGE_SIZE") if can_sample_proc else None + except (OSError, ValueError): + can_sample_proc = False + page_size = None + + if not can_sample_proc: + try: + subprocess.run(cmd, check=True, **_sp_kwargs) + except subprocess.CalledProcessError: + _report_memory(wheel, "finished", None, None) + raise + _report_memory(wheel, "finished", None, None) + return + + previous_sigterm_handler = signal.signal(signal.SIGTERM, _exit_on_sigterm) + process = None + + try: + process = subprocess.Popen( + cmd, + start_new_session=True, + **_sp_kwargs, + ) + peak = 0 + next_report = _REPORT_STEP_BYTES + + while True: + if can_sample_proc: + try: + current = _process_tree_rss_bytes(process.pid, page_size) + except (OSError, ValueError): + can_sample_proc = False + peak = None + else: + peak = max(peak, current) + if peak >= next_report: + _report_memory(wheel, "running", current, peak) + next_report = ( + peak // _REPORT_STEP_BYTES + 1 + ) * _REPORT_STEP_BYTES + + returncode = process.poll() + if returncode is not None: + break + time.sleep(_SAMPLE_INTERVAL_SECONDS) + except BaseException: + if process is not None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + raise + finally: + signal.signal(signal.SIGTERM, previous_sigterm_handler) + + _report_memory(wheel, "finished", None, peak or None) + if returncode: + raise subprocess.CalledProcessError(returncode, cmd) diff --git a/uv/private/pep517_whl/memory_monitor_test.py b/uv/private/pep517_whl/memory_monitor_test.py new file mode 100644 index 000000000..d885b7b8b --- /dev/null +++ b/uv/private/pep517_whl/memory_monitor_test.py @@ -0,0 +1,219 @@ +import contextlib +import io +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + +from uv.private.pep517_whl import memory_monitor + + +class ProcessTreeRssTest(unittest.TestCase): + def test_sums_descendants_spawned_by_any_task(self): + with tempfile.TemporaryDirectory() as directory: + proc_root = Path(directory) + self._write_process(proc_root, 10, 2, {10: [20], 11: [30]}) + self._write_process(proc_root, 20, 3, {20: []}) + self._write_process(proc_root, 30, 5, {30: []}) + + self.assertEqual( + 10 * 4096, + memory_monitor._process_tree_rss_bytes(10, 4096, str(proc_root)), + ) + + def test_missing_children_file_warns_and_covers_only_root(self): + """When children files are absent (CONFIG_PROC_CHILDREN missing), the walk + stops at the root PID and emits a one-time warning to stderr.""" + stderr = io.StringIO() + with tempfile.TemporaryDirectory() as directory: + proc_root = Path(directory) + + process_dir = proc_root / "10" + process_dir.mkdir() + (process_dir / "statm").write_text("100 5 0 0 0 0 0\n") + (process_dir / "task" / "10").mkdir(parents=True) + child_dir = proc_root / "20" + child_dir.mkdir() + (child_dir / "statm").write_text("100 3 0 0 0 0 0\n") + + with ( + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor, "_children_unavailable_warned", False), + ): + result = memory_monitor._process_tree_rss_bytes(10, 4096, str(proc_root)) + + self.assertEqual(5 * 4096, result) # only root PID; process 20 unreachable + self.assertIn("CONFIG_PROC_CHILDREN", stderr.getvalue()) + + def test_success_without_monitoring_uses_run(self): + with tempfile.TemporaryFile() as stdout: + memory_monitor.run_with_memory_profile( + [sys.executable, "-c", "pass"], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=False, + ) + + def test_failed_child_reports_memory_before_raising(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor, "_REPORT_STEP_BYTES", 1), + ): + with self.assertRaisesRegex(subprocess.CalledProcessError, "exit status 7"): + memory_monitor.run_with_memory_profile( + [ + sys.executable, + "-c", + "import time; data = bytearray(16 * 1024 * 1024); time.sleep(.5); raise SystemExit(7)", + ], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + report = stderr.getvalue() + if sys.platform.startswith("linux") and os.path.isdir("/proc"): + self.assertRegex( + report, + r"wheel build memory for test-wheel[.]tar[.]gz running: sampled aggregate current=[1-9][0-9.]* MiB, peak=[1-9][0-9.]* MiB", + ) + self.assertRegex( + report, + r"wheel build memory for test-wheel[.]tar[.]gz finished: sampled aggregate peak=[1-9][0-9.]* MiB", + ) + else: + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + report, + ) + + def test_sampler_failure_reports_unavailable(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor.sys, "platform", "linux"), + mock.patch.object(memory_monitor.os.path, "isdir", return_value=True), + mock.patch.object( + memory_monitor, + "_process_tree_rss_bytes", + side_effect=OSError("procfs unavailable"), + ), + ): + memory_monitor.run_with_memory_profile( + [sys.executable, "-c", "import time; time.sleep(.1)"], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + stderr.getvalue(), + ) + + def test_unsupported_platform_uses_run(self): + stderr = io.StringIO() + with ( + tempfile.TemporaryFile() as stdout, + contextlib.redirect_stderr(stderr), + mock.patch.object(memory_monitor.sys, "platform", "win32"), + mock.patch.object(memory_monitor.subprocess, "run") as run, + ): + memory_monitor.run_with_memory_profile( + ["command"], + cwd=None, + env={}, + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + run.assert_called_once() + self.assertIn( + "wheel build memory for test-wheel.tar.gz finished: unavailable", + stderr.getvalue(), + ) + + @unittest.skipUnless( + sys.platform.startswith("linux") and os.path.isdir("/proc"), + "requires Linux procfs", + ) + def test_exception_kills_and_waits_for_process_tree(self): + for interrupt_signal, expected_exception in ( + (signal.SIGINT, KeyboardInterrupt), + (signal.SIGTERM, SystemExit), + ): + with ( + self.subTest(signal=interrupt_signal), + tempfile.TemporaryDirectory() as directory, + tempfile.TemporaryFile() as stdout, + ): + pids_file = Path(directory) / "pids" + with self.assertRaises(expected_exception) as raised: + memory_monitor.run_with_memory_profile( + [ + sys.executable, + "-c", + "import os, pathlib, signal, subprocess, sys, time; " + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)']); " + "pathlib.Path(sys.argv[1]).write_text(f'{os.getpid()} {child.pid}'); " + "os.kill(os.getppid(), int(sys.argv[2])); " + "time.sleep(60)", + str(pids_file), + str(interrupt_signal), + ], + cwd=None, + env=os.environ.copy(), + stdout=stdout, + wheel="test-wheel.tar.gz", + monitor=True, + ) + + if interrupt_signal == signal.SIGTERM: + self.assertEqual(128 + signal.SIGTERM, raised.exception.code) + + frontend_pid, child_pid = [ + int(pid) for pid in pids_file.read_text().split() + ] + for pid in (frontend_pid, child_pid): + deadline = time.monotonic() + 5 + while True: + try: + os.kill(pid, 0) + except ProcessLookupError: + break + if time.monotonic() >= deadline: + self.fail( + "process {} survived interruption cleanup".format(pid) + ) + time.sleep(0.01) + + @staticmethod + def _write_process(proc_root, pid, rss_pages, task_children): + process_dir = proc_root / str(pid) + process_dir.mkdir() + (process_dir / "statm").write_text("100 {} 0 0 0 0 0\n".format(rss_pages)) + for task_id, children in task_children.items(): + task_dir = process_dir / "task" / str(task_id) + task_dir.mkdir(parents=True) + (task_dir / "children").write_text( + " ".join(str(child) for child in children) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv/private/pep517_whl/rule.bzl b/uv/private/pep517_whl/rule.bzl index 1fb57149e..3c05b3ad8 100644 --- a/uv/private/pep517_whl/rule.bzl +++ b/uv/private/pep517_whl/rule.bzl @@ -5,9 +5,10 @@ Uses `python -m build` (the pypa/build frontend) which delegates to whatever build backend the sdist declares in its `[build-system]` table. """ -load("@bazel_lib//lib:resource_sets.bzl", "resource_set", "resource_set_attr") +load("@bazel_lib//lib:resource_sets.bzl", "resource_set", "resource_set_attr", "resource_set_for") load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") load("//py/private/toolchain:types.bzl", "NATIVE_BUILD_TOOLCHAIN", "PY_TOOLCHAIN") +load(":build_memory.bzl", "validate_build_memory_mb") def _common_env(ctx): return { @@ -29,6 +30,15 @@ def _patch_args_and_inputs(ctx): patch_inputs.append(f) return patch_args, patch_inputs +def _memory_args(ctx): + return ["--monitor-memory"] if ctx.attr.build_memory_mb else [] + +def _build_resource_set(ctx): + validate_build_memory_mb(ctx.attr.build_memory_mb, ctx.label) + if ctx.attr.build_memory_mb: + return resource_set_for(mem_mb = ctx.attr.build_memory_mb) + return resource_set(ctx.attr) + def _collect_toolchain_inputs_and_vars(ctx): """Gather files + Make-variable substitutions from `ctx.attr.toolchains`. @@ -97,7 +107,7 @@ def _cc_toolchain_inputs_and_compiler(ctx): def _pep517_whl(ctx): archive = ctx.file.src - wheel_dir = ctx.actions.declare_directory("whl") + wheel_dir = ctx.actions.declare_directory(ctx.label.name) patch_args, patch_inputs = _patch_args_and_inputs(ctx) # The build tool is a py_binary wrapping build_helper.py. Using it as @@ -110,7 +120,7 @@ def _pep517_whl(ctx): progress_message = "Source compiling {} to a whl".format(archive.basename), executable = ctx.executable.tool, toolchain = None, - arguments = ctx.attr.args + patch_args + [ + arguments = ctx.attr.args + patch_args + _memory_args(ctx) + [ archive.path, wheel_dir.path, ], @@ -119,14 +129,14 @@ def _pep517_whl(ctx): outputs = [wheel_dir], env = _common_env(ctx), exec_group = "target", - resource_set = resource_set(ctx.attr), + resource_set = _build_resource_set(ctx), ) return [DefaultInfo(files = depset([wheel_dir]))] def _pep517_native_whl(ctx): archive = ctx.file.src - wheel_dir = ctx.actions.declare_directory("whl") + wheel_dir = ctx.actions.declare_directory(ctx.label.name) patch_args, patch_inputs = _patch_args_and_inputs(ctx) env = _common_env(ctx) @@ -148,7 +158,7 @@ def _pep517_native_whl(ctx): progress_message = "Native source compiling {} to a whl".format(archive.basename), executable = ctx.executable.tool, toolchain = None, - arguments = ctx.attr.args + patch_args + [ + arguments = ctx.attr.args + patch_args + _memory_args(ctx) + [ archive.path, wheel_dir.path, ], @@ -160,7 +170,7 @@ def _pep517_native_whl(ctx): outputs = [wheel_dir], env = env, exec_group = "target", - resource_set = resource_set(ctx.attr), + resource_set = _build_resource_set(ctx), ) return [DefaultInfo(files = depset([wheel_dir]))] @@ -178,6 +188,12 @@ _PATCH_ATTRS = { } _pep517_whl_attrs = { + "build_memory_mb": attr.int( + default = 0, + doc = "Estimated peak memory in MB for local wheel builds, from 0 to " + + "32768. Bazel rounds this up to the next resource class supported " + + "by bazel_lib. Zero uses Bazel's default estimate.", + ), "src": attr.label(allow_single_file = True), "tool": attr.label(executable = True, cfg = "exec"), "version": attr.string(), diff --git a/uv/private/pep517_whl/test.bzl b/uv/private/pep517_whl/test.bzl index 9f9b758cb..510a6a900 100644 --- a/uv/private/pep517_whl/test.bzl +++ b/uv/private/pep517_whl/test.bzl @@ -17,6 +17,26 @@ _JDK_ENV_KEYS = ["JAVA_HOME", "JAVA", "JAR"] _REQUIRED_ENV_KEYS = _CC_ENV_KEYS + _JDK_ENV_KEYS +def _analysis_failure_test_impl(ctx): + env = analysistest.begin(ctx) + asserts.expect_failure(env, ctx.attr.expected_error) + return analysistest.end(env) + +build_memory_failure_test = analysistest.make( + _analysis_failure_test_impl, + attrs = {"expected_error": attr.string()}, + expect_failure = True, +) + +def _build_memory_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + actions = [action for action in target.actions if action.mnemonic == "PySdistBuild"] + asserts.equals(env, 1, len(actions)) + return analysistest.end(env) + +build_memory_test = analysistest.make(_build_memory_test_impl) + def _toolchain_env_test_impl(ctx): env = analysistest.begin(ctx) target = analysistest.target_under_test(env) diff --git a/uv/private/sdist_build/repository.bzl b/uv/private/sdist_build/repository.bzl index 8ceb77b3c..0f3cdc8e1 100644 --- a/uv/private/sdist_build/repository.bzl +++ b/uv/private/sdist_build/repository.bzl @@ -208,6 +208,9 @@ def _sdist_build_impl(repository_ctx): # Merge explicit deps with auto-discovered deps all_deps = [str(d) for d in repository_ctx.attr.deps] + extra_dep_labels + build_tool_deps = [ + "@aspect_rules_py//uv/private/pep517_whl:memory_monitor", + ] + all_deps pre_build_patches = repository_ctx.attr.pre_build_patches patch_attrs = "" @@ -259,6 +262,10 @@ def _sdist_build_impl(repository_ctx): if repository_ctx.attr.resource_set != "default": resource_set_attr = "\n resource_set = \"{}\",".format(repository_ctx.attr.resource_set) + memory_attr = "" + if repository_ctx.attr.build_memory_mb: + memory_attr = "\n build_memory_mb = {},".format(repository_ctx.attr.build_memory_mb) + repository_ctx.file("BUILD.bazel", content = """ load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "{rule}") load("@aspect_rules_py//py:defs.bzl", "py_binary") @@ -275,7 +282,7 @@ py_binary( src = "{src}", tool = ":build_tool", version = "{version}", - args = [],{resource_set_attr}{patch_attrs}{toolchain_attrs} + args = [],{resource_set_attr}{memory_attr}{patch_attrs}{toolchain_attrs} visibility = ["//visibility:public"], ) @@ -285,10 +292,11 @@ exports_files( ) """.format( src = repository_ctx.attr.src, - deps = repr(all_deps), + deps = repr(build_tool_deps), rule = "pep517_native_whl" if is_native else "pep517_whl", version = repository_ctx.attr.version, resource_set_attr = resource_set_attr, + memory_attr = memory_attr, patch_attrs = patch_attrs, toolchain_attrs = toolchain_attrs, )) @@ -319,6 +327,10 @@ sdist_build = repository_rule( "`resource_set` attribute, reserving local RAM/CPU for the wheel build " + "action. Set via `uv.override_package(resource_set = ...)`.", ), + "build_memory_mb": attr.int( + default = 0, + doc = "Estimated peak memory in MB for local wheel builds, from 0 to 32768. Set via uv.override_package(build_memory_mb = ...).", + ), "pre_build_patches": attr.label_list(default = []), "pre_build_patch_strip": attr.int(default = 0), "extra_toolchains": attr.string_list( diff --git a/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel b/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel index d8ca27afe..54d448b05 100644 --- a/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel +++ b/uv/private/uv_hub/snapshots/sdist_build.bravado_core.BUILD.bazel @@ -6,7 +6,7 @@ py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@@+uv+project__aspect_rules_py//:build", "@@+uv+project__aspect_rules_py//:setuptools", "@@+uv+project__aspect_rules_py//:maturin", "@whl_install__aspect_rules_py__setuptools__80_9_0//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@+uv+project__aspect_rules_py//:build", "@@+uv+project__aspect_rules_py//:setuptools", "@@+uv+project__aspect_rules_py//:maturin", "@whl_install__aspect_rules_py__setuptools__80_9_0//:install"], ) pep517_whl( From ae003a68ca3eac64b762652ff8e6ee8b68337346 Mon Sep 17 00:00:00 2001 From: xangcastle Date: Mon, 22 Jun 2026 23:12:44 -0600 Subject: [PATCH 3/6] resolve merge conflicts --- docs/uv-patching.md | 19 ++++------ e2e/BUILD.bazel | 10 +++--- .../uv-sdist-jdk-build/setup.MODULE.bazel | 5 ++- ...uild.uv_sdist_jdk_build.jpype1.BUILD.bazel | 1 - uv/private/extension/BUILD.bazel | 1 - uv/private/extension/defs.bzl | 19 ++-------- uv/private/pep517_whl/BUILD.bazel | 36 ------------------- uv/private/pep517_whl/build_memory.bzl | 12 ------- uv/private/pep517_whl/rule.bzl | 21 +++-------- uv/private/pep517_whl/test.bzl | 20 ----------- uv/private/sdist_build/repository.bzl | 11 +----- 11 files changed, 20 insertions(+), 135 deletions(-) delete mode 100644 uv/private/pep517_whl/build_memory.bzl diff --git a/docs/uv-patching.md b/docs/uv-patching.md index 2732eaad5..033f6fa33 100644 --- a/docs/uv-patching.md +++ b/docs/uv-patching.md @@ -83,27 +83,22 @@ Pre-build patches are applied to the extracted source tree after archive extract - Removing problematic native build dependencies - Patching source code that affects the build output -### Allocating wheel build memory +### Measuring wheel build memory -Set `build_memory_mb` to the expected peak memory of a local sdist build so -Bazel can limit concurrent wheel builds: +On Linux, when `resource_set` is set, rules_py samples the aggregate resident +memory of the build process and its descendants and reports the peak to stderr: ```starlark uv.override_package( lock = "//:uv.lock", name = "native-package", - build_memory_mb = 6144, + resource_set = "mem_2g", ) ``` -Bazel rounds the estimate up to one of its supported resource classes. On -Linux, setting a nonzero estimate also enables aggregate resident-memory -sampling for the build process and its descendants. Estimates above 32768 MiB -are rejected because bazel-lib 3.2 would otherwise clamp them to 32768 MiB and -silently under-reserve the action. A measurement appears whenever the sampled -peak crosses another 256 MiB threshold and once when the build finishes. If -procfs is unavailable or cannot be read, the final measurement is reported as -unavailable. +A measurement appears whenever the sampled peak crosses another 256 MiB +threshold and once when the build finishes. If procfs is unavailable or cannot +be read, the final measurement is reported as unavailable. ### Adding extra dependencies or data diff --git a/e2e/BUILD.bazel b/e2e/BUILD.bazel index f88936b6e..be5da9667 100644 --- a/e2e/BUILD.bazel +++ b/e2e/BUILD.bazel @@ -110,14 +110,12 @@ write_source_files( # shows up as the per-wheel dicts collapsing into one merged list. "snapshots/abi3_compat.whl_install.cffi.BUILD.bazel": "@whl_install__abi3_compat__cffi__2_0_0//:BUILD.bazel", - # @sdist_build for jpype1 with uv.override_package build resources, - # toolchains, and env + # @sdist_build for jpype1 with uv.override_package toolchains and env # ---------------------------------------------------------------- # Covers the override_package -> repo-rule -> BUILD-template - # plumbing for `build_memory_mb`, `toolchains = [...]`, and - # `env = {...}`. The override layers a JDK runtime on top of the - # default CC toolchain; the snapshot pins the memory declaration, - # toolchain references, and both env families (CC + JAVA_HOME/JAVA/JAR). + # plumbing for `toolchains = [...]` and `env = {...}`. The override + # layers a JDK runtime on top of the default CC toolchain; the snapshot + # pins the toolchain references and both env families (CC + JAVA_HOME/JAVA/JAR). # Pairs with the analysis test at # //uv/private/pep517_whl:toolchain_env_test which asserts the # rule expands env keys correctly given an explicit fixture. diff --git a/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel b/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel index e696558bc..ece40aa71 100644 --- a/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel +++ b/e2e/cases/uv-sdist-jdk-build/setup.MODULE.bazel @@ -23,11 +23,10 @@ uv.unstable_annotate_packages( lock = "//cases/uv-sdist-jdk-build:uv.lock", ) -# Reserve local build memory and layer a JDK runtime toolchain into the build -# env. `toolchains` and `env` are additive to sdist_build's defaults. +# Layer a JDK runtime toolchain into the build env. +# `toolchains` and `env` are additive to sdist_build's defaults. uv.override_package( name = "jpype1", - build_memory_mb = 2048, env = { "JAR": "$(JAVABASE)/bin/jar", "JAVA": "$(JAVA)", diff --git a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel index 4e25a55c9..f61aedd6b 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel @@ -15,7 +15,6 @@ pep517_native_whl( tool = ":build_tool", version = "1.7.1", args = [], - build_memory_mb = 2048, toolchains = [ "@bazel_tools//tools/cpp:current_cc_toolchain", "@@bazel_tools//tools/jdk:current_java_runtime", diff --git a/uv/private/extension/BUILD.bazel b/uv/private/extension/BUILD.bazel index 768403783..2ac723012 100644 --- a/uv/private/extension/BUILD.bazel +++ b/uv/private/extension/BUILD.bazel @@ -22,7 +22,6 @@ bzl_library( "//uv/private:normalize_version", "//uv/private/constraints:repository", "//uv/private/git_archive:repository", - "//uv/private/pep517_whl:build_memory", "//uv/private/pprint:defs", "//uv/private/sdist_build:repository", "//uv/private/sdist_configure:defs", diff --git a/uv/private/extension/defs.bzl b/uv/private/extension/defs.bzl index f4fcb2cfc..08494ce53 100644 --- a/uv/private/extension/defs.bzl +++ b/uv/private/extension/defs.bzl @@ -61,7 +61,6 @@ load("//uv/private:normalize_version.bzl", "normalize_version") load("//uv/private:parse_whl_name.bzl", "parse_whl_name") load("//uv/private/constraints:repository.bzl", "configurations_hub") load("//uv/private/git_archive:repository.bzl", "git_archive") -load("//uv/private/pep517_whl:build_memory.bzl", "validate_build_memory_mb") load("//uv/private/pprint:defs.bzl", "pprint") load("//uv/private/sdist_build:repository.bzl", "sdist_build") load("//uv/private/sdist_configure:defs.bzl", "DEFAULT_CONFIGURE_SCRIPT") @@ -248,10 +247,6 @@ def _parse_projects(module_ctx, hub_specs): )) has_target = override.target != None - validate_build_memory_mb( - override.build_memory_mb, - "uv.override_package() for '{}'".format(override.name), - ) has_modifications = ( override.pre_build_patches or override.post_install_patches or @@ -259,15 +254,14 @@ def _parse_projects(module_ctx, hub_specs): override.extra_data or override.toolchains or override.env or - override.resource_set != "default" or - override.build_memory_mb + override.resource_set != "default" ) if has_target and has_modifications: fail("uv.override_package() for '{}': `target` is mutually exclusive with modification attributes. Use `target` for full replacement OR build, patch, and data attributes for modifications, not both.".format(override.name)) if not has_target and not has_modifications: - fail("uv.override_package() for '{}': must specify either `target` for full replacement or at least one modification attribute (pre_build_patches, post_install_patches, extra_deps, extra_data, toolchains, env, resource_set, build_memory_mb).".format(override.name)) + fail("uv.override_package() for '{}': must specify either `target` for full replacement or at least one modification attribute (pre_build_patches, post_install_patches, extra_deps, extra_data, toolchains, env, resource_set).".format(override.name)) package_overrides[override_key] = override @@ -419,12 +413,10 @@ def _parse_projects(module_ctx, hub_specs): extra_toolchains = [] extra_env = {} resource_set = "default" - build_memory_mb = 0 if pkg_override: extra_toolchains = [str(t) for t in pkg_override.toolchains] extra_env = pkg_override.env resource_set = pkg_override.resource_set - build_memory_mb = pkg_override.build_memory_mb sbuild_specs[sbuild_id] = struct( src = sdist, @@ -438,7 +430,6 @@ def _parse_projects(module_ctx, hub_specs): extra_toolchains = extra_toolchains, extra_env = extra_env, resource_set = resource_set, - build_memory_mb = build_memory_mb, ) has_sbuild = True @@ -668,8 +659,6 @@ def _uv_impl(module_ctx): sbuild_kwargs["extra_env"] = sbuild_cfg.extra_env if sbuild_cfg.resource_set != "default": sbuild_kwargs["resource_set"] = sbuild_cfg.resource_set - if sbuild_cfg.build_memory_mb: - sbuild_kwargs["build_memory_mb"] = sbuild_cfg.build_memory_mb sdist_build(**sbuild_kwargs) for install_id, install_cfg in cfg.install_cfgs.items(): @@ -787,10 +776,6 @@ _override_package_tag = tag_class( # CC/CXX/AR/LD/STRIP env) — they don't replace them. Use these # to layer extra toolchains (Java runtime, Rust, …) and extra # env vars on top of the defaults. - "build_memory_mb": attr.int( - default = 0, - doc = "Estimated peak memory in MB for this package's local wheel build, from 0 to 32768. Bazel rounds up to a supported resource class; zero uses its default estimate.", - ), "toolchains": attr.label_list( default = [], doc = "Extra toolchain targets appended to the generated pep517_native_whl(...) call's `toolchains` list. Each target's TemplateVariableInfo make-variables become available for $(VAR) expansion in `env`.", diff --git a/uv/private/pep517_whl/BUILD.bazel b/uv/private/pep517_whl/BUILD.bazel index f5e84cb1e..bd337f4fa 100644 --- a/uv/private/pep517_whl/BUILD.bazel +++ b/uv/private/pep517_whl/BUILD.bazel @@ -5,8 +5,6 @@ load("//py:defs.bzl", "py_library", "py_test") load(":rule.bzl", "pep517_native_whl", "pep517_whl") load( ":test.bzl", - "build_memory_failure_test", - "build_memory_test", "pep517_native_whl_toolchain_env_test", ) @@ -23,7 +21,6 @@ bzl_library( name = "rule", srcs = ["rule.bzl"], deps = [ - ":build_memory", "//py/private/toolchain:types", "@bazel_lib//lib:resource_sets", ], @@ -82,35 +79,6 @@ sh_binary( tags = ["manual"], ) -pep517_whl( - name = "__excessive_memory_fixture", - src = ":__stub_sdist", - build_memory_mb = 32769, - tags = ["manual"], - tool = ":__stub_tool", - version = "0.0.1", -) - -build_memory_failure_test( - name = "build_memory_limit_test", - expected_error = "build_memory_mb must be between 0 and 32768", - target_under_test = ":__excessive_memory_fixture", -) - -pep517_whl( - name = "__maximum_memory_fixture", - src = ":__stub_sdist", - build_memory_mb = 32768, - tags = ["manual"], - tool = ":__stub_tool", - version = "0.0.1", -) - -build_memory_test( - name = "build_memory_maximum_test", - target_under_test = ":__maximum_memory_fixture", -) - # Fixture replicates the toolchains + env defaults emitted by sdist_build's # BUILD template (see uv/private/sdist_build/repository.bzl). The analysis # test below asserts these expand to non-empty toolchain paths. @@ -141,7 +109,3 @@ pep517_native_whl_toolchain_env_test( target_under_test = ":__toolchain_env_fixture", ) -bzl_library( - name = "build_memory", - srcs = ["build_memory.bzl"], -) diff --git a/uv/private/pep517_whl/build_memory.bzl b/uv/private/pep517_whl/build_memory.bzl deleted file mode 100644 index 17297ede9..000000000 --- a/uv/private/pep517_whl/build_memory.bzl +++ /dev/null @@ -1,12 +0,0 @@ -"""Validation shared by wheel build configuration surfaces.""" - -# bazel-lib 3.2 clamps larger resource requests to 32 GiB: -# https://github.com/aspect-build/bazel-lib/blob/v3.2.0/lib/private/resource_sets.bzl#L2224-L2227 -MAX_BUILD_MEMORY_MB = 32768 - -def validate_build_memory_mb(value, owner): - if value < 0 or value > MAX_BUILD_MEMORY_MB: - fail("{}: build_memory_mb must be between 0 and {}".format( - owner, - MAX_BUILD_MEMORY_MB, - )) diff --git a/uv/private/pep517_whl/rule.bzl b/uv/private/pep517_whl/rule.bzl index 3c05b3ad8..40ff8e0aa 100644 --- a/uv/private/pep517_whl/rule.bzl +++ b/uv/private/pep517_whl/rule.bzl @@ -5,10 +5,9 @@ Uses `python -m build` (the pypa/build frontend) which delegates to whatever build backend the sdist declares in its `[build-system]` table. """ -load("@bazel_lib//lib:resource_sets.bzl", "resource_set", "resource_set_attr", "resource_set_for") +load("@bazel_lib//lib:resource_sets.bzl", "resource_set", "resource_set_attr") load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") load("//py/private/toolchain:types.bzl", "NATIVE_BUILD_TOOLCHAIN", "PY_TOOLCHAIN") -load(":build_memory.bzl", "validate_build_memory_mb") def _common_env(ctx): return { @@ -31,13 +30,7 @@ def _patch_args_and_inputs(ctx): return patch_args, patch_inputs def _memory_args(ctx): - return ["--monitor-memory"] if ctx.attr.build_memory_mb else [] - -def _build_resource_set(ctx): - validate_build_memory_mb(ctx.attr.build_memory_mb, ctx.label) - if ctx.attr.build_memory_mb: - return resource_set_for(mem_mb = ctx.attr.build_memory_mb) - return resource_set(ctx.attr) + return ["--monitor-memory"] if ctx.attr.resource_set != "default" else [] def _collect_toolchain_inputs_and_vars(ctx): """Gather files + Make-variable substitutions from `ctx.attr.toolchains`. @@ -129,7 +122,7 @@ def _pep517_whl(ctx): outputs = [wheel_dir], env = _common_env(ctx), exec_group = "target", - resource_set = _build_resource_set(ctx), + resource_set = resource_set(ctx.attr), ) return [DefaultInfo(files = depset([wheel_dir]))] @@ -170,7 +163,7 @@ def _pep517_native_whl(ctx): outputs = [wheel_dir], env = env, exec_group = "target", - resource_set = _build_resource_set(ctx), + resource_set = resource_set(ctx.attr), ) return [DefaultInfo(files = depset([wheel_dir]))] @@ -188,12 +181,6 @@ _PATCH_ATTRS = { } _pep517_whl_attrs = { - "build_memory_mb": attr.int( - default = 0, - doc = "Estimated peak memory in MB for local wheel builds, from 0 to " + - "32768. Bazel rounds this up to the next resource class supported " + - "by bazel_lib. Zero uses Bazel's default estimate.", - ), "src": attr.label(allow_single_file = True), "tool": attr.label(executable = True, cfg = "exec"), "version": attr.string(), diff --git a/uv/private/pep517_whl/test.bzl b/uv/private/pep517_whl/test.bzl index 510a6a900..9f9b758cb 100644 --- a/uv/private/pep517_whl/test.bzl +++ b/uv/private/pep517_whl/test.bzl @@ -17,26 +17,6 @@ _JDK_ENV_KEYS = ["JAVA_HOME", "JAVA", "JAR"] _REQUIRED_ENV_KEYS = _CC_ENV_KEYS + _JDK_ENV_KEYS -def _analysis_failure_test_impl(ctx): - env = analysistest.begin(ctx) - asserts.expect_failure(env, ctx.attr.expected_error) - return analysistest.end(env) - -build_memory_failure_test = analysistest.make( - _analysis_failure_test_impl, - attrs = {"expected_error": attr.string()}, - expect_failure = True, -) - -def _build_memory_test_impl(ctx): - env = analysistest.begin(ctx) - target = analysistest.target_under_test(env) - actions = [action for action in target.actions if action.mnemonic == "PySdistBuild"] - asserts.equals(env, 1, len(actions)) - return analysistest.end(env) - -build_memory_test = analysistest.make(_build_memory_test_impl) - def _toolchain_env_test_impl(ctx): env = analysistest.begin(ctx) target = analysistest.target_under_test(env) diff --git a/uv/private/sdist_build/repository.bzl b/uv/private/sdist_build/repository.bzl index 0f3cdc8e1..1be47608c 100644 --- a/uv/private/sdist_build/repository.bzl +++ b/uv/private/sdist_build/repository.bzl @@ -262,10 +262,6 @@ def _sdist_build_impl(repository_ctx): if repository_ctx.attr.resource_set != "default": resource_set_attr = "\n resource_set = \"{}\",".format(repository_ctx.attr.resource_set) - memory_attr = "" - if repository_ctx.attr.build_memory_mb: - memory_attr = "\n build_memory_mb = {},".format(repository_ctx.attr.build_memory_mb) - repository_ctx.file("BUILD.bazel", content = """ load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "{rule}") load("@aspect_rules_py//py:defs.bzl", "py_binary") @@ -282,7 +278,7 @@ py_binary( src = "{src}", tool = ":build_tool", version = "{version}", - args = [],{resource_set_attr}{memory_attr}{patch_attrs}{toolchain_attrs} + args = [],{resource_set_attr}{patch_attrs}{toolchain_attrs} visibility = ["//visibility:public"], ) @@ -296,7 +292,6 @@ exports_files( rule = "pep517_native_whl" if is_native else "pep517_whl", version = repository_ctx.attr.version, resource_set_attr = resource_set_attr, - memory_attr = memory_attr, patch_attrs = patch_attrs, toolchain_attrs = toolchain_attrs, )) @@ -327,10 +322,6 @@ sdist_build = repository_rule( "`resource_set` attribute, reserving local RAM/CPU for the wheel build " + "action. Set via `uv.override_package(resource_set = ...)`.", ), - "build_memory_mb": attr.int( - default = 0, - doc = "Estimated peak memory in MB for local wheel builds, from 0 to 32768. Set via uv.override_package(build_memory_mb = ...).", - ), "pre_build_patches": attr.label_list(default = []), "pre_build_patch_strip": attr.int(default = 0), "extra_toolchains": attr.string_list( From a7a6f59c1d4a53a530a493f528ef004d6ece3016 Mon Sep 17 00:00:00 2001 From: xangcastle Date: Mon, 22 Jun 2026 23:22:48 -0600 Subject: [PATCH 4/6] fix buildifier and snapshots --- e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel | 1 - ...sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel | 2 +- uv/private/extension/BUILD.bazel | 1 - uv/private/pep517_whl/BUILD.bazel | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel index 4e25a55c9..f61aedd6b 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel @@ -15,7 +15,6 @@ pep517_native_whl( tool = ":build_tool", version = "1.7.1", args = [], - build_memory_mb = 2048, toolchains = [ "@bazel_tools//tools/cpp:current_cc_toolchain", "@@bazel_tools//tools/jdk:current_java_runtime", diff --git a/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel index 6cbd37583..505501e11 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel @@ -6,7 +6,7 @@ py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@@aspect_rules_py++uv+project__uv_sdist_native_build//:build", "@@aspect_rules_py++uv+project__uv_sdist_native_build//:setuptools", "@whl_install__uv_sdist_native_build__setuptools__75_8_2//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@aspect_rules_py++uv+project__uv_sdist_native_build//:build", "@@aspect_rules_py++uv+project__uv_sdist_native_build//:setuptools", "@whl_install__uv_sdist_native_build__setuptools__75_8_2//:install"], ) pep517_native_whl( diff --git a/uv/private/extension/BUILD.bazel b/uv/private/extension/BUILD.bazel index 768403783..2ac723012 100644 --- a/uv/private/extension/BUILD.bazel +++ b/uv/private/extension/BUILD.bazel @@ -22,7 +22,6 @@ bzl_library( "//uv/private:normalize_version", "//uv/private/constraints:repository", "//uv/private/git_archive:repository", - "//uv/private/pep517_whl:build_memory", "//uv/private/pprint:defs", "//uv/private/sdist_build:repository", "//uv/private/sdist_configure:defs", diff --git a/uv/private/pep517_whl/BUILD.bazel b/uv/private/pep517_whl/BUILD.bazel index bd337f4fa..1d2cca0c5 100644 --- a/uv/private/pep517_whl/BUILD.bazel +++ b/uv/private/pep517_whl/BUILD.bazel @@ -108,4 +108,3 @@ pep517_native_whl_toolchain_env_test( name = "toolchain_env_test", target_under_test = ":__toolchain_env_fixture", ) - From fb29717b6e23367c99ac61ccf7734ed5aef9a37b Mon Sep 17 00:00:00 2001 From: xangcastle Date: Mon, 22 Jun 2026 23:25:06 -0600 Subject: [PATCH 5/6] fix gazelle --- uv/private/pep517_whl/BUILD.bazel | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uv/private/pep517_whl/BUILD.bazel b/uv/private/pep517_whl/BUILD.bazel index 1d2cca0c5..26a84e5bf 100644 --- a/uv/private/pep517_whl/BUILD.bazel +++ b/uv/private/pep517_whl/BUILD.bazel @@ -108,3 +108,8 @@ pep517_native_whl_toolchain_env_test( name = "toolchain_env_test", target_under_test = ":__toolchain_env_fixture", ) + +bzl_library( + name = "build_memory", + srcs = ["build_memory.bzl"], +) From e481a2b9034eb91249b352a3b19a10b35bbd26d1 Mon Sep 17 00:00:00 2001 From: xangcastle Date: Mon, 22 Jun 2026 23:30:51 -0600 Subject: [PATCH 6/6] fix Signals.SIGIN --- uv/private/pep517_whl/memory_monitor_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv/private/pep517_whl/memory_monitor_test.py b/uv/private/pep517_whl/memory_monitor_test.py index d885b7b8b..776eea555 100644 --- a/uv/private/pep517_whl/memory_monitor_test.py +++ b/uv/private/pep517_whl/memory_monitor_test.py @@ -174,7 +174,7 @@ def test_exception_kills_and_waits_for_process_tree(self): "os.kill(os.getppid(), int(sys.argv[2])); " "time.sleep(60)", str(pids_file), - str(interrupt_signal), + str(int(interrupt_signal)), ], cwd=None, env=os.environ.copy(),