From 66b46bd127200f4b7be1a0f8fa860f5565b4d84a Mon Sep 17 00:00:00 2001 From: xangcastle Date: Tue, 7 Jul 2026 02:16:23 -0600 Subject: [PATCH 1/2] reproducing package collision --- e2e/cases/MODULE.bazel | 1 + .../namespace-init-collision/BUILD.bazel | 14 ++++++++ .../namespace-init-collision/__test__.py | 17 +++++++++ .../namespace-init-collision/pyproject.toml | 8 +++++ .../setup.MODULE.bazel | 8 +++++ e2e/cases/namespace-init-collision/uv.lock | 36 +++++++++++++++++++ 6 files changed, 84 insertions(+) create mode 100644 e2e/cases/namespace-init-collision/BUILD.bazel create mode 100644 e2e/cases/namespace-init-collision/__test__.py create mode 100644 e2e/cases/namespace-init-collision/pyproject.toml create mode 100644 e2e/cases/namespace-init-collision/setup.MODULE.bazel create mode 100644 e2e/cases/namespace-init-collision/uv.lock diff --git a/e2e/cases/MODULE.bazel b/e2e/cases/MODULE.bazel index fcd99dfac..1375bc71d 100644 --- a/e2e/cases/MODULE.bazel +++ b/e2e/cases/MODULE.bazel @@ -99,6 +99,7 @@ include("//freethreaded-805:setup.MODULE.bazel") include("//inactive-marker:setup.MODULE.bazel") include("//interpreter-local-pyvenv-home:setup.MODULE.bazel") include("//multi-project-hub:setup.MODULE.bazel") +include("//namespace-init-collision:setup.MODULE.bazel") include("//oci:setup.MODULE.bazel") include("//pbs-cc-toolchain:setup.MODULE.bazel") include("//pth-namespace-547:setup.MODULE.bazel") diff --git a/e2e/cases/namespace-init-collision/BUILD.bazel b/e2e/cases/namespace-init-collision/BUILD.bazel new file mode 100644 index 000000000..1688975a0 --- /dev/null +++ b/e2e/cases/namespace-init-collision/BUILD.bazel @@ -0,0 +1,14 @@ +load("@aspect_rules_py//py:defs.bzl", "py_test") + +py_test( + name = "test", + srcs = ["__test__.py"], + dep_group = "namespace-init-collision", + main = "__test__.py", + package_collisions = "error", + python_version = "3.11", + deps = [ + "@pypi_namespace_init_collision//backports_shutil_get_terminal_size", + "@pypi_namespace_init_collision//backports_weakref", + ], +) diff --git a/e2e/cases/namespace-init-collision/__test__.py b/e2e/cases/namespace-init-collision/__test__.py new file mode 100644 index 000000000..b689fd4b4 --- /dev/null +++ b/e2e/cases/namespace-init-collision/__test__.py @@ -0,0 +1,17 @@ +"""Regression test for namespace package __init__.py stub collisions. + +Both backports.weakref and backports.shutil-get-terminal-size install a +backports/__init__.py that declares a pkgutil namespace package. The files are +not byte-identical, but they are semantically equivalent, so the venv must not +treat them as a collision. +""" + + +def test_namespace_imports(): + import backports.shutil_get_terminal_size # noqa: F401 + import backports.weakref # noqa: F401 + + +if __name__ == "__main__": + test_namespace_imports() + print("PASS: namespace package stubs did not collide") diff --git a/e2e/cases/namespace-init-collision/pyproject.toml b/e2e/cases/namespace-init-collision/pyproject.toml new file mode 100644 index 000000000..f7682bd7f --- /dev/null +++ b/e2e/cases/namespace-init-collision/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "namespace-init-collision" +version = "0.0.0" +requires-python = ">=3.11" +dependencies = [ + "backports.weakref", + "backports.shutil-get-terminal-size", +] diff --git a/e2e/cases/namespace-init-collision/setup.MODULE.bazel b/e2e/cases/namespace-init-collision/setup.MODULE.bazel new file mode 100644 index 000000000..e2d2bc8c8 --- /dev/null +++ b/e2e/cases/namespace-init-collision/setup.MODULE.bazel @@ -0,0 +1,8 @@ +uv = use_extension("@aspect_rules_py//uv:extensions.bzl", "uv") +uv.declare_hub(hub_name = "pypi_namespace_init_collision") +uv.project( + hub_name = "pypi_namespace_init_collision", + lock = "//namespace-init-collision:uv.lock", + pyproject = "//namespace-init-collision:pyproject.toml", +) +use_repo(uv, "pypi_namespace_init_collision") diff --git a/e2e/cases/namespace-init-collision/uv.lock b/e2e/cases/namespace-init-collision/uv.lock new file mode 100644 index 000000000..9d083bb79 --- /dev/null +++ b/e2e/cases/namespace-init-collision/uv.lock @@ -0,0 +1,36 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "backports-shutil-get-terminal-size" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz", hash = "sha256:713e7a8228ae80341c70586d1cc0a8caa5207346927e23d09dcbcaf18eadec80", size = 4279, upload-time = "2014-08-19T18:42:49.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/cd/1750d6c35fe86d35f8562091737907f234b78fdffab42b29c72b1dd861f4/backports.shutil_get_terminal_size-1.0.0-py2.py3-none-any.whl", hash = "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64", size = 6497, upload-time = "2014-08-19T18:42:51.908Z" }, +] + +[[package]] +name = "backports-weakref" +version = "1.0.post1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/12/ab/cf35cf43a4a6215e3255cf2e49c77d5ba1e9c733af2aa3ec1ca9c4d02592/backports.weakref-1.0.post1.tar.gz", hash = "sha256:bc4170a29915f8b22c9e7c4939701859650f2eb84184aee80da329ac0b9825c2", size = 10574, upload-time = "2017-09-17T20:58:21.892Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/ec/f598b633c3d5ffe267aaada57d961c94fdfa183c5c3ebda2b6d151943db6/backports.weakref-1.0.post1-py2.py3-none-any.whl", hash = "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82", size = 5237, upload-time = "2017-09-17T20:58:26.557Z" }, +] + +[[package]] +name = "namespace-init-collision" +version = "0.0.0" +source = { virtual = "." } +dependencies = [ + { name = "backports-shutil-get-terminal-size" }, + { name = "backports-weakref" }, +] + +[package.metadata] +requires-dist = [ + { name = "backports-shutil-get-terminal-size" }, + { name = "backports-weakref" }, +] From 57025ec907d2251785c12d1ecc6923763a1fdefd Mon Sep 17 00:00:00 2001 From: xangcastle Date: Tue, 7 Jul 2026 13:48:01 -0600 Subject: [PATCH 2/2] fix - reproducing package collision --- py/private/providers.bzl | 7 +- py/private/py_unpacked_wheel.bzl | 15 +- py/private/py_venv/venv.bzl | 16 ++- py/tools/site_merge/site_merge.py | 192 ++++++++++++++++++++++++- py/tools/site_merge/site_merge_test.py | 61 ++++++++ uv/private/whl_install/repository.bzl | 103 ++++++++++--- uv/private/whl_install/rule.bzl | 32 ++++- 7 files changed, 392 insertions(+), 34 deletions(-) diff --git a/py/private/providers.bzl b/py/private/providers.bzl index 817ce769f..325203efe 100644 --- a/py/private/providers.bzl +++ b/py/private/providers.bzl @@ -20,7 +20,12 @@ handling gives the later distinct element in the flattened sequence precedence. Duplicate dependency edges do not create another precedence position. Fields: * `top_levels`: tuple[str] — complete set of immediate `site-packages` entry names when nonempty; an empty tuple means the layout is unknown. - * `namespace_top_levels`: tuple[str] — subset of top_levels that are PEP 420 namespace packages. + * `namespace_top_levels`: tuple[str] — subset of top_levels that are PEP 420 namespace packages + (no `/__init__.py`) or pkgutil-style namespace packages (an `__init__.py` that + only calls `pkgutil.extend_path` / `pkg_resources.declare_namespace`). + * `pkgutil_namespace_top_levels`: tuple[str] — subset of `namespace_top_levels` that carry a + pkgutil-style `__init__.py`. venv assembly symlinks this `__init__.py` into the merged + namespace directory; may be absent on structs from older producers. * `namespace_entries`: tuple[str] — `/`-joined paths of the concrete entries beneath the namespace top-levels (e.g. `jaraco/functools`), used to materialise a merged namespace directory out of per-entry symlinks. May be absent on structs from diff --git a/py/private/py_unpacked_wheel.bzl b/py/private/py_unpacked_wheel.bzl index 14736bb49..982e380fa 100644 --- a/py/private/py_unpacked_wheel.bzl +++ b/py/private/py_unpacked_wheel.bzl @@ -83,6 +83,7 @@ def _py_unpacked_wheel_impl(ctx): wheels = depset(direct = [struct( top_levels = tuple(ctx.attr.top_levels), namespace_top_levels = tuple(ctx.attr.namespace_top_levels), + pkgutil_namespace_top_levels = tuple(ctx.attr.pkgutil_namespace_top_levels), namespace_entries = tuple(ctx.attr.namespace_entries), site_packages_rfpath = site_packages_rfpath, console_scripts = tuple(ctx.attr.console_scripts), @@ -128,12 +129,24 @@ wrappers under `/bin/`. Typically populated from the wheel's default = [], ), "namespace_top_levels": attr.string_list( - doc = """Subset of `top_levels` that are PEP 420 namespace packages. + doc = """Subset of `top_levels` that are namespace packages. +Includes both PEP 420 namespaces (no `__init__.py`) and +pkgutil/pkg_resources-style namespaces (an `__init__.py` that only +calls `pkgutil.extend_path` or `pkg_resources.declare_namespace`). See the equivalent attribute on the `whl_install` rule for the full story; short version: names listed here suppress collision errors when multiple wheels claim the same top-level, because Python's namespace machinery is meant to merge their contributions. +""", + default = [], + ), + "pkgutil_namespace_top_levels": attr.string_list( + doc = """Subset of `namespace_top_levels` that carry a pkgutil-style `__init__.py`. + +These top-levels are merged across wheels like PEP 420 namespaces, but +venv assembly also symlinks the stub `__init__.py` into the merged +directory so the namespace package is importable at runtime. """, default = [], ), diff --git a/py/private/py_venv/venv.bzl b/py/private/py_venv/venv.bzl index 99a7603e0..97d4c397a 100644 --- a/py/private/py_venv/venv.bzl +++ b/py/private/py_venv/venv.bzl @@ -136,13 +136,14 @@ def _resolve_wheel_collisions(ctx, wheels, package_collisions): # Pass 1: bucket claimants per import root, distribution metadata entry, # and console-script name. - tl_claimants = {} # tl -> list of struct(site_packages, is_ns, ns_entries) + tl_claimants = {} # tl -> list of struct(site_packages, is_ns, is_pkgutil_ns, ns_entries) metadata_claimants = {} # metadata entry -> ordered site_packages paths cs_claimants = {} # name -> list of struct(site_packages, module, func) wheel_by_sp = {} # site_packages_rfpath -> wheel struct for w in wheels: wheel_by_sp[w.site_packages_rfpath] = w ns_set = {tl: True for tl in getattr(w, "namespace_top_levels", ())} + pkgutil_ns_set = {tl: True for tl in getattr(w, "pkgutil_namespace_top_levels", ())} ns_entries_by_tl = {} for entry in getattr(w, "namespace_entries", ()): ns_entries_by_tl.setdefault(entry.split("/")[0], []).append(entry) @@ -153,6 +154,7 @@ def _resolve_wheel_collisions(ctx, wheels, package_collisions): tl_claimants.setdefault(tl, []).append(struct( site_packages = w.site_packages_rfpath, is_ns = tl in ns_set, + is_pkgutil_ns = tl in pkgutil_ns_set, ns_entries = tuple(ns_entries_by_tl.get(tl, [])), )) for entry in getattr(w, "console_scripts", ()): @@ -251,6 +253,11 @@ def _resolve_wheel_collisions(ctx, wheels, package_collisions): entry_owner[entry] = c for entry, c in entry_owner.items(): top_level_to_site_pkgs[entry] = c.site_packages + + for c in unique_claimants: + if c.is_pkgutil_ns: + top_level_to_site_pkgs[tl + "/__init__.py"] = c.site_packages + break continue # Pure PEP 420 namespace (no regular package spanning wheels). @@ -327,6 +334,13 @@ def _resolve_wheel_collisions(ctx, wheels, package_collisions): for entry, c in entry_owner.items(): top_level_to_site_pkgs[entry] = c.site_packages + init_owner = None + for c in unique_claimants: + if c.is_pkgutil_ns: + init_owner = c + if init_owner: + top_level_to_site_pkgs[tl + "/__init__.py"] = init_owner.site_packages + # Entries-bearing claimants that kept every one of their # entries are fully represented by the merged directory; record # per-wheel coverage so pass 3 can drop them from the .pth diff --git a/py/tools/site_merge/site_merge.py b/py/tools/site_merge/site_merge.py index 71755926c..c1782e13f 100644 --- a/py/tools/site_merge/site_merge.py +++ b/py/tools/site_merge/site_merge.py @@ -24,6 +24,7 @@ """ import argparse +import ast import filecmp import os import shutil @@ -32,6 +33,195 @@ from pathlib import Path +def _is_namespace_init_ast(tree): + """Return True if the AST is a recognized namespace package declaration. + + Accepts the canonical forms generated by setuptools/pkgutil namespace + packages. Any executable code beyond the namespace declaration returns + False so that real collisions are still surfaced. + """ + # Unwrap a Module body. + body = tree.body if isinstance(tree, ast.Module) else [tree] + + # Drop any docstring/Expr(Str) statements that may appear in legacy stubs. + meaningful = [] + import_seen = False + for stmt in body: + if ( + isinstance(stmt, ast.Expr) and + isinstance(stmt.value, ast.Constant) and + isinstance(stmt.value.value, str) + ): + continue + # Track the canonical `from pkgutil import extend_path` import, then + # expect the assignment as the only other meaningful statement. + if isinstance(stmt, ast.ImportFrom) and stmt.module == "pkgutil": + if len(stmt.names) == 1 and stmt.names[0].name == "extend_path": + import_seen = True + continue + meaningful.append(stmt) + + if len(meaningful) != 1: + return False + stmt = meaningful[0] + + # pkgutil form 1: __path__ = __import__("pkgutil").extend_path(__path__, __name__) + if isinstance(stmt, ast.Assign): + if len(stmt.targets) != 1: + return False + target = stmt.targets[0] + if not (isinstance(target, ast.Name) and target.id == "__path__"): + return False + return _is_pkgutil_extend_path_call(stmt.value) + + # pkg_resources form: __import__("pkg_resources").declare_namespace(__name__) + if isinstance(stmt, ast.Expr): + return _is_pkg_resources_declare_namespace_call(stmt.value) + + # try/except fallback form: + # try: + # __import__("pkg_resources").declare_namespace(__name__) + # except ImportError: + # __import__("pkgutil").extend_path(__path__, __name__) + if isinstance(stmt, ast.Try): + if len(stmt.body) != 1 or len(stmt.handlers) != 1: + return False + if not ( + stmt.handlers[0].type is None and + len(stmt.handlers[0].body) == 1 + ): + return False + return ( + _is_pkg_resources_declare_namespace_call(stmt.body[0].value) and + _is_pkgutil_extend_path_call(stmt.handlers[0].body[0].value) + ) + + return False + + +def _is_pkgutil_extend_path_call(node): + """Return True if node is pkgutil.extend_path(__path__, __name__) or equivalent.""" + if not isinstance(node, ast.Call): + return False + func = node.func + + # pkgutil form 2 (module attribute): import pkgutil; + # __path__ = pkgutil.extend_path(__path__, __name__) + if ( + isinstance(func, ast.Attribute) and + func.attr == "extend_path" and + isinstance(func.value, ast.Name) and + func.value.id == "pkgutil" + ): + return _is_extend_path_args(node.args) + + # pkgutil form 3 (name-loaded call): from pkgutil import extend_path; + # __path__ = extend_path(__path__, __name__) + if ( + isinstance(func, ast.Name) and + func.id == "extend_path" + ): + return _is_extend_path_args(node.args) + + # __import__("pkgutil").extend_path + if ( + isinstance(func, ast.Attribute) and + func.attr == "extend_path" and + isinstance(func.value, ast.Call) + ): + call = func.value + if ( + isinstance(call.func, ast.Name) and + call.func.id == "__import__" and + len(call.args) == 1 and + isinstance(call.args[0], ast.Constant) and + call.args[0].value == "pkgutil" + ): + return _is_extend_path_args(node.args) + + return False + + +def _is_extend_path_args(args): + """Return True if args are (__path__, __name__).""" + if len(args) != 2: + return False + return ( + isinstance(args[0], ast.Name) and args[0].id == "__path__" and + isinstance(args[1], ast.Name) and args[1].id == "__name__" + ) + + +def _is_pkg_resources_declare_namespace_call(node): + """Return True if node is pkg_resources.declare_namespace(__name__) or equivalent.""" + if not isinstance(node, ast.Call): + return False + func = node.func + + # pkg_resources.declare_namespace + if ( + isinstance(func, ast.Attribute) and + func.attr == "declare_namespace" and + isinstance(func.value, ast.Name) and + func.value.id == "pkg_resources" + ): + return _is_declare_namespace_args(node.args) + + # __import__("pkg_resources").declare_namespace + if ( + isinstance(func, ast.Attribute) and + func.attr == "declare_namespace" and + isinstance(func.value, ast.Call) + ): + call = func.value + if ( + isinstance(call.func, ast.Name) and + call.func.id == "__import__" and + len(call.args) == 1 and + isinstance(call.args[0], ast.Constant) and + call.args[0].value == "pkg_resources" + ): + return _is_declare_namespace_args(node.args) + + return False + + +def _is_declare_namespace_args(args): + """Return True if args are (__name__,).""" + return ( + len(args) == 1 and + isinstance(args[0], ast.Name) and + args[0].id == "__name__" + ) + + +def _is_namespace_init_stub(content): + """Return True if the __init__.py content is a namespace package stub.""" + try: + tree = ast.parse(content) + except SyntaxError: + return False + return _is_namespace_init_ast(tree) + + +def _are_files_equivalent(src_file, dest): + """Compare two files, treating namespace stubs as equivalent. + + Returns True if the files are byte-identical or if both are + __init__.py files containing only a recognized namespace package stub. + """ + if filecmp.cmp(str(src_file), str(dest), shallow=False): + return True + if src_file.name != "__init__.py" or dest.name != "__init__.py": + return False + src_content = src_file.read_text() + dest_content = dest.read_text() + return ( + _is_namespace_init_stub(src_content) and + _is_namespace_init_stub(dest_content) + ) + + def _remove(path): """Remove an output copied from a potentially read-only input.""" @@ -85,7 +275,7 @@ def merge(into, sources): # The wheel extractor treats any execute bit as executable # (py/tools/unpack/unpack.py). Other mode differences may # reflect executor umask and are benign for identical data. - if filecmp.cmp(str(src_file), str(dest), shallow=False): + if _are_files_equivalent(src_file, dest): src_executable = bool(src_file.stat().st_mode & 0o111) dest_executable = bool(dest.stat().st_mode & 0o111) if src_executable != dest_executable: diff --git a/py/tools/site_merge/site_merge_test.py b/py/tools/site_merge/site_merge_test.py index 16b172565..c86a44aa5 100644 --- a/py/tools/site_merge/site_merge_test.py +++ b/py/tools/site_merge/site_merge_test.py @@ -124,6 +124,67 @@ def test_collision_policy_controls_reporting_and_status(self): if policy != "error": self.assertEqual((output / "entry/child.py").read_text(), "second") + def test_namespace_init_stubs_treated_as_equivalent(self): + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + first = root / "first" + second = root / "second" + output = root / "output" + + _write( + first / "backports/__init__.py", + "from pkgutil import extend_path\n__path__ = extend_path(__path__, __name__)\n", + ) + _write( + first / "backports/weakref.py", + "VALUE = 'first'\n", + ) + _write( + second / "backports/__init__.py", + "# See https://pypi.python.org/pypi/backports\n\n" + "from pkgutil import extend_path\n" + "__path__ = extend_path(__path__, __name__)\n", + ) + _write( + second / "backports/shutil_get_terminal_size.py", + "VALUE = 'second'\n", + ) + + conflicts = merge(output, [first, second]) + + self.assertEqual(conflicts, []) + self.assertEqual( + (output / "backports/weakref.py").read_text(), + "VALUE = 'first'\n", + ) + self.assertEqual( + (output / "backports/shutil_get_terminal_size.py").read_text(), + "VALUE = 'second'\n", + ) + + def test_namespace_init_stub_with_extra_code_is_collision(self): + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + first = root / "first" + second = root / "second" + output = root / "output" + + _write( + first / "backports/__init__.py", + "from pkgutil import extend_path\n__path__ = extend_path(__path__, __name__)\n", + ) + _write( + second / "backports/__init__.py", + "from pkgutil import extend_path\n__path__ = extend_path(__path__, __name__)\nX = 1\n", + ) + + conflicts = merge(output, [first, second]) + + self.assertEqual( + {path.name for path, _previous, _current in conflicts}, + {"__init__.py"}, + ) + if __name__ == "__main__": unittest.main() diff --git a/uv/private/whl_install/repository.bzl b/uv/private/whl_install/repository.bzl index 244f5d1de..7c9e36469 100644 --- a/uv/private/whl_install/repository.bzl +++ b/uv/private/whl_install/repository.bzl @@ -144,6 +144,28 @@ def _whl_file_label_keys(whl_file_label): return keys +def _is_pkgutil_namespace_init(content): + """Return True if an __init__.py only declares a pkgutil/pkg_resources namespace. + + pkgutil-style namespace packages carry an `__init__.py` that calls + `pkgutil.extend_path(__path__, __name__)`; setuptools-style namespace + packages call `pkg_resources.declare_namespace(__name__)`. Both are + semantically namespace declarations: multiple wheels can legitimately + ship the same `__init__.py` for the same top-level, and the venv should + merge the directories rather than treating the top-level as a collision. + """ + normalized = [] + for line in content.splitlines(): + # Strip comments and whitespace. + line = line.split("#", 1)[0].strip() + if line: + normalized.append(line) + normalized = " ".join(normalized) + return ( + ("extend_path" in normalized and "__path__" in normalized) or + ("declare_namespace" in normalized and "__name__" in normalized) + ) + def _extract_wheel_metadata(repository_ctx, whl_label): """Peek inside a wheel to discover top-level names and console scripts. @@ -154,6 +176,9 @@ def _extract_wheel_metadata(repository_ctx, whl_label): Reads: * `*.dist-info/RECORD` (mandatory per PEP 427) to get top-level names. * `*.dist-info/entry_points.txt` (optional) to get `[console_scripts]`. + * every `/__init__.py` to detect pkgutil/pkg_resources-style + namespace packages, which are then treated like PEP 420 namespaces for + collision purposes. Fails if the archive cannot be inspected, so package collision and console-script handling never silently depend on host tooling. @@ -165,22 +190,26 @@ def _extract_wheel_metadata(repository_ctx, whl_label): label_list attr so Bazel wires up repo visibility. Returns: - Tuple (top_levels_set, regular_top_levels_set, console_scripts_set, - namespace_entries_set, dirs_set, init_dirs_set): + Tuple (top_levels_set, regular_top_levels_set, pkgutil_namespace_top_levels_set, + console_scripts_set, namespace_entries_set, dirs_set, init_dirs_set): * top_levels_set: dict[name → True] — all first-path-segment names in RECORD (excluding `*.data/` staging entries). * regular_top_levels_set: subset that had an `__init__.py` at - depth 1, i.e. regular packages. Its complement (within - top_levels_set, minus `.dist-info/`) is the PEP 420 namespace - set for this wheel. + depth 1 which is NOT a pkgutil/pkg_resources namespace stub. + * pkgutil_namespace_top_levels_set: subset that had an + `/__init__.py` declaring a pkgutil/pkg_resources namespace. + These top-levels are also considered namespace packages and are + excluded from `regular_top_levels_set`. * console_scripts_set: dict[script_name → "name=module:func"]. * namespace_entries_set: dict[path → True] — for each top-level - that looks like a PEP 420 namespace in THIS wheel, the - `/`-joined paths of the concrete entries beneath it: the - shallowest directory holding a direct `__init__.py` (recursing - through nested namespace dirs like `google/cloud/`), or the - file itself for plain modules / data files. Lets venv assembly - materialise a merged namespace dir out of per-entry symlinks. + that looks like a namespace in THIS wheel, the `/`-joined paths + of the concrete entries beneath it: the shallowest directory + holding a direct `__init__.py` (recursing through nested namespace + dirs like `google/cloud/`), or the file itself for plain modules / + data files. Lets venv assembly materialise a merged namespace dir + out of per-entry symlinks. The stub `__init__.py` of a pkgutil + namespace top-level is intentionally omitted here; venv assembly + symlinks it separately. * dirs_set: dict[path → True] — every directory implied by a RECORD entry, as a `/`-joined relative path. * init_dirs_set: dict[path → True] — subset of dirs_set that @@ -216,11 +245,11 @@ def _extract_wheel_metadata(repository_ctx, whl_label): repository_ctx.extract( archive = metadata_archive, output = metadata_dir, - strip_prefix = metadata_directory, ) repository_ctx.delete(metadata_archive) metadata_path = repository_ctx.path(metadata_dir) - record_path = metadata_path.get_child("RECORD") + dist_info_path = metadata_path.get_child(metadata_directory) + record_path = dist_info_path.get_child("RECORD") if not record_path.exists: fail("{}: wheel {} has no {}/RECORD".format( repository_ctx.name, @@ -229,20 +258,19 @@ def _extract_wheel_metadata(repository_ctx, whl_label): )) record = repository_ctx.read(record_path) entry_points = "" - entry_points_path = metadata_path.get_child("entry_points.txt") + entry_points_path = dist_info_path.get_child("entry_points.txt") if entry_points_path.exists: entry_points = repository_ctx.read(entry_points_path) - repository_ctx.delete(metadata_dir) data_directory = metadata_directory[:-len(".dist-info")] + ".data" # RECORD: authoritative list of every installed file. First path segment # = top-level name after translating wheel install-scheme paths. top_levels_set = {} - # Tracks which top-levels contain a direct `/__init__.py` — - # i.e., are regular packages. The complement (top_levels that never - # appear with an `__init__.py` at depth 1) are PEP 420 namespace - # packages that expect to be merged across wheels. + # Tracks which top-levels contain a direct `/__init__.py`. + # A top-level is "regular" only if that `__init__.py` is NOT a pkgutil + # or pkg_resources namespace stub; otherwise it is tracked in + # pkgutil_namespace_top_levels below. regular_top_levels = {} # Raw material for the namespace derivations below: every kept RECORD @@ -291,6 +319,17 @@ def _extract_wheel_metadata(repository_ctx, whl_label): if len(segments) >= 2 and segments[-1] == "__init__.py": init_dirs["/".join(segments[:-1])] = True + pkgutil_namespace_top_levels = {} + for tl in list(regular_top_levels.keys()): + init_path = metadata_path.get_child(tl).get_child("__init__.py") + if init_path.exists: + content = repository_ctx.read(init_path) + if _is_pkgutil_namespace_init(content): + pkgutil_namespace_top_levels[tl] = True + regular_top_levels.pop(tl, None) + + repository_ctx.delete(metadata_dir) + # Namespace entries: for each path under a (per-this-wheel) namespace # top-level, descend until hitting the shallowest concrete prefix — a # directory with a direct `__init__.py`, or the file itself when no @@ -299,6 +338,9 @@ def _extract_wheel_metadata(repository_ctx, whl_label): # (`google/cloud/storage/…`) recurse naturally: `google/cloud` has no # `__init__.py`, so the walk continues to `google/cloud/storage`. # Entries for top-levels that turn out regular are filtered out here. + # For pkgutil namespace top-levels, the stub `__init__.py` itself is + # intentionally excluded; venv assembly symlinks it separately so the + # namespace directory is importable. namespace_entries = {} for segments in record_segments: if segments[0] in regular_top_levels or segments[0].endswith(".dist-info"): @@ -308,6 +350,11 @@ def _extract_wheel_metadata(repository_ctx, whl_label): for depth in range(2, len(segments) + 1): prefix = "/".join(segments[:depth]) if depth == len(segments) or prefix in init_dirs: + if ( + segments[0] in pkgutil_namespace_top_levels and + prefix == segments[0] + "/__init__.py" + ): + break namespace_entries[prefix] = True break @@ -335,7 +382,7 @@ def _extract_wheel_metadata(repository_ctx, whl_label): name, normalised = entry console_scripts[name] = normalised - return whl_path.basename, top_levels_set, regular_top_levels, console_scripts, namespace_entries, dirs_set, init_dirs + return whl_path.basename, top_levels_set, regular_top_levels, pkgutil_namespace_top_levels, console_scripts, namespace_entries, dirs_set, init_dirs def _namespace_dirs_and_roots(dirs_set, init_dirs, namespace_top_levels_set): """Split a wheel's directory skeleton into the implicit-namespace dirs @@ -696,6 +743,7 @@ filegroup( top_levels_by_whl = {} namespace_top_levels_by_whl = {} + pkgutil_namespace_top_levels_by_whl = {} namespace_entries_by_whl = {} namespace_dirs_by_whl = {} regular_roots_by_whl = {} @@ -710,15 +758,16 @@ filegroup( continue extracted_targets[target] = True whl_file_label = whl_file_labels[target] - whl_name, tls, regular, css, ns_entries, dirs_set, init_dirs = _extract_wheel_metadata( + whl_name, tls, regular, pkgutil_ns, css, ns_entries, dirs_set, init_dirs = _extract_wheel_metadata( repository_ctx, whl_file_label, ) if tls: top_levels_by_whl[whl_name] = sorted(tls.keys()) - # A top-level counts as a PEP 420 namespace for this wheel if - # its RECORD shows no `/__init__.py` at depth 1. + # A top-level counts as a namespace for this wheel if its RECORD + # shows no regular `/__init__.py` at depth 1. pkgutil + # namespace stubs are also treated as namespace packages. namespaces = sorted([ tl for tl in tls @@ -730,6 +779,9 @@ filegroup( namespace_top_levels_by_whl[whl_name] = namespaces namespace_set = {tl: True for tl in namespaces} + if pkgutil_ns: + pkgutil_namespace_top_levels_by_whl[whl_name] = sorted(pkgutil_ns.keys()) + # Concrete entries beneath this wheel's namespace # top-levels (`jaraco/functools`) — for the per-entry # symlink merge that makes the namespace mypy/pyright @@ -770,6 +822,11 @@ filegroup( # Only emitted for wheels that contribute to a namespace — keeps the # generated BUILD files (and their e2e snapshots) unchanged for the # common regular-top-level-only case. + if pkgutil_namespace_top_levels_by_whl: + install_attrs += """ + pkgutil_namespace_top_levels = {pkgutil_namespace_top_levels},""".format( + pkgutil_namespace_top_levels = indent(pprint(pkgutil_namespace_top_levels_by_whl), " " * 4).lstrip(), + ) if namespace_entries_by_whl: install_attrs += """ namespace_entries = {namespace_entries},""".format( diff --git a/uv/private/whl_install/rule.bzl b/uv/private/whl_install/rule.bzl index 9cf75bbd8..799d4baa5 100644 --- a/uv/private/whl_install/rule.bzl +++ b/uv/private/whl_install/rule.bzl @@ -59,6 +59,7 @@ def _whl_install(ctx): if SourceBuiltWheelInfo in ctx.attr.src: top_levels = [] namespace_top_levels = [] + pkgutil_namespace_top_levels = [] namespace_entries = [] namespace_dirs = [] regular_roots = [] @@ -67,6 +68,7 @@ def _whl_install(ctx): whl_basename = archive.basename top_levels = ctx.attr.top_levels.get(whl_basename, []) namespace_top_levels = ctx.attr.namespace_top_levels.get(whl_basename, []) + pkgutil_namespace_top_levels = ctx.attr.pkgutil_namespace_top_levels.get(whl_basename, []) namespace_entries = ctx.attr.namespace_entries.get(whl_basename, []) namespace_dirs = ctx.attr.namespace_dirs.get(whl_basename, []) regular_roots = ctx.attr.regular_roots.get(whl_basename, []) @@ -177,6 +179,12 @@ def _whl_install(ctx): # physical merge instead. Falls back to .pth-based # resolution when entry metadata is missing. namespace_top_levels = tuple(namespace_top_levels), + # pkgutil/pkg_resources-style namespace packages this wheel + # contributes to. Like PEP 420 namespaces, these are merged + # across wheels; unlike PEP 420 namespaces, the stub + # `__init__.py` is symlinked into the merged directory so + # the namespace package is importable. + pkgutil_namespace_top_levels = tuple(pkgutil_namespace_top_levels), # Concrete per-wheel paths beneath namespace top-levels # (e.g. `jaraco/functools`) that venv assembly symlinks # individually to materialise a merged namespace directory. @@ -279,14 +287,24 @@ under `/bin/` so `subprocess.run(["", ...])` works. default = {}, ), "namespace_top_levels": attr.string_list_dict( - doc = """Per-wheel subset of `top_levels` that are PEP 420 namespace packages, keyed by wheel file basename. + doc = """Per-wheel subset of `top_levels` that are namespace packages, keyed by wheel file basename. -A top-level is a namespace if the wheel's RECORD shows no -`/__init__.py`. When multiple wheels contribute to the same -namespace (e.g. `jaraco-classes` and `jaraco-functools` both claim -`jaraco`), `py_binary`'s collision detector treats the overlap as -benign and falls back to `.pth`-based resolution so Python's namespace -machinery merges the contributions at runtime. +A top-level is a namespace if the wheel's RECORD shows no regular +`/__init__.py`. pkgutil/pkg_resources-style namespace packages +(those whose `__init__.py` only calls `pkgutil.extend_path` or +`pkg_resources.declare_namespace`) are also included. When multiple wheels +contribute to the same namespace (e.g. `jaraco-classes` and +`jaraco-functools` both claim `jaraco`), `py_binary`'s collision detector +treats the overlap as benign and merges the contributions concretely. +""", + default = {}, + ), + "pkgutil_namespace_top_levels": attr.string_list_dict( + doc = """Per-wheel subset of `namespace_top_levels` that carry a pkgutil-style `__init__.py`, keyed by wheel file basename. + +These top-levels are merged across wheels like PEP 420 namespaces, but +venv assembly also symlinks the stub `__init__.py` into the merged +directory so the namespace package is importable at runtime. """, default = {}, ),