From 6b788ea62641ddd0bba8b1cc8586728c63cd6e55 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 21 Jul 2025 07:49:00 +0900 Subject: [PATCH 1/6] feat(pypi): add a standards compliant python_tag creator This will be needed when we start selecting wheels entirely in the bzlmod extension evaluation phase (#3058). This adds a few unit tests to just ensure that we conform to the spec even though the code is very simple. Work towards #2747 Work towards #2759 Work towards #2849 --- python/private/pypi/BUILD.bazel | 8 +++++ python/private/pypi/python_tag.bzl | 41 ++++++++++++++++++++++ tests/pypi/python_tag/BUILD.bazel | 3 ++ tests/pypi/python_tag/python_tag_tests.bzl | 34 ++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 python/private/pypi/python_tag.bzl create mode 100644 tests/pypi/python_tag/BUILD.bazel create mode 100644 tests/pypi/python_tag/python_tag_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index b098f29e94..6ae8aeca52 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -338,6 +338,14 @@ bzl_library( ], ) +bzl_library( + name = "python_tag_bzl", + srcs = ["python_tag.bzl"], + deps = [ + "//python/private:version_bzl", + ], +) + bzl_library( name = "render_pkg_aliases_bzl", srcs = ["render_pkg_aliases.bzl"], diff --git a/python/private/pypi/python_tag.bzl b/python/private/pypi/python_tag.bzl new file mode 100644 index 0000000000..224c5f96f0 --- /dev/null +++ b/python/private/pypi/python_tag.bzl @@ -0,0 +1,41 @@ +"A simple utility function to get the python_tag from the implementation name" + +load("//python/private:version.bzl", "version") + +# Taken from +# https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#python-tag +_PY_TAGS = { + # "py": Generic Python (does not require implementation-specific features) + "cpython": "cp", + "ironpython": "ip", + "jython": "jy", + "pypy": "pp", + "python": "py", +} +PY_TAG_GENERIC = "py" + +def python_tag(implementation_name, python_version = ""): + """Get the python_tag from the implementation_name. + + Args: + implementation_name: {type}`str` the implementation name, e.g. "cpython" + python_version: {type}`str` a version who can be parsed using PEP440 compliant + parser. + + Returns: + A {type}`str` that represents the python_tag with a version if the + python_version is given. + """ + if python_version: + v = version.parse(python_version, strict = True) + suffix = "{}{}".format( + v.release[0], + v.release[1] if len(v.release) > 1 else "", + ) + else: + suffix = "" + + return "{}{}".format( + _PY_TAGS.get(implementation_name, implementation_name), + suffix, + ) diff --git a/tests/pypi/python_tag/BUILD.bazel b/tests/pypi/python_tag/BUILD.bazel new file mode 100644 index 0000000000..d4b37cea16 --- /dev/null +++ b/tests/pypi/python_tag/BUILD.bazel @@ -0,0 +1,3 @@ +load(":python_tag_tests.bzl", "python_tag_test_suite") + +python_tag_test_suite(name = "python_tag_tests") diff --git a/tests/pypi/python_tag/python_tag_tests.bzl b/tests/pypi/python_tag/python_tag_tests.bzl new file mode 100644 index 0000000000..ca86575e5b --- /dev/null +++ b/tests/pypi/python_tag/python_tag_tests.bzl @@ -0,0 +1,34 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:python_tag.bzl", "python_tag") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_without_version(env): + for give, expect in { + "cpython": "cp", + "ironpython": "ip", + "jython": "jy", + "pypy": "pp", + "python": "py", + "something_else": "something_else", + }.items(): + got = python_tag(give) + env.expect.that_str(got).equals(expect) + +_tests.append(_test_without_version) + +def _test_with_version(env): + got = python_tag("cpython", "3.1.15") + env.expect.that_str(got).equals("cp31") + +_tests.append(_test_with_version) + +def python_tag_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 247e0659f08aeacadcb7ea2335004721941abcae Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:09:22 +0900 Subject: [PATCH 2/6] feat(pypi): implement a new whl selection algorithm DO NOT MERGE: stacked on #3110 This PR only implements the selection algorithm where instead of selecting all wheels that are compatible with the set of target platforms, we select a single wheel that is most specialized for a particular *single* target platform. What is more, compared to the existing algorithm it does not assume a particular list of supported platforms and just fully implements the spec. Work towards #2747 Work towards #2759 Work towards #2849 --- python/private/pypi/BUILD.bazel | 10 + python/private/pypi/select_whl.bzl | 230 ++++++++++ tests/pypi/select_whl/BUILD.bazel | 3 + tests/pypi/select_whl/select_whl_tests.bzl | 463 +++++++++++++++++++++ 4 files changed, 706 insertions(+) create mode 100644 python/private/pypi/select_whl.bzl create mode 100644 tests/pypi/select_whl/BUILD.bazel create mode 100644 tests/pypi/select_whl/select_whl_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 6ae8aeca52..f854d4edcc 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -367,6 +367,16 @@ bzl_library( ], ) +bzl_library( + name = "select_whl_bzl", + srcs = ["select_whl.bzl"], + deps = [ + ":parse_whl_name_bzl", + ":python_tag_bzl", + "//python/private:version_bzl", + ], +) + bzl_library( name = "simpleapi_download_bzl", srcs = ["simpleapi_download.bzl"], diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl new file mode 100644 index 0000000000..91268d6735 --- /dev/null +++ b/python/private/pypi/select_whl.bzl @@ -0,0 +1,230 @@ +"Select a single wheel that fits the parameters of a target platform." + +load("//python/private:version.bzl", "version") +load(":parse_whl_name.bzl", "parse_whl_name") +load(":python_tag.bzl", "PY_TAG_GENERIC", "python_tag") + +_ANDROID = "android" +_IOS = "ios" +_MANYLINUX = "manylinux" +_MACOSX = "macosx" +_MUSLLINUX = "musllinux" + +def _value_priority(*, tag, values): + keys = [] + for priority, wp in enumerate(values): + if tag == wp: + keys.append(priority) + + return max(keys) if keys else None + +def _platform_tag_priority(*, tag, values): + # Implements matching platform tag + # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ + + if not ( + tag.startswith(_ANDROID) or + tag.startswith(_IOS) or + tag.startswith(_MACOSX) or + tag.startswith(_MANYLINUX) or + tag.startswith(_MUSLLINUX) + ): + res = _value_priority(tag = tag, values = values) + if res == None: + return res + + return (res, (0, 0)) + + plat, _, tail = tag.partition("_") + major, _, tail = tail.partition("_") + if not plat.startswith(_ANDROID): + minor, _, arch = tail.partition("_") + else: + minor = "0" + arch = tail + version = (int(major), int(minor)) + + keys = [] + for priority, wp in enumerate(values): + want_plat, sep, tail = wp.partition("_") + if not sep: + continue + + if want_plat != plat: + continue + + want_major, _, tail = tail.partition("_") + if want_major == "*": + want_major = "" + want_minor = "" + want_arch = tail + elif plat.startswith(_ANDROID): + want_minor = "0" + want_arch = tail + else: + want_minor, _, want_arch = tail.partition("_") + + if want_arch != arch: + continue + + want_version = (int(want_major), int(want_minor)) if want_major else None + if not want_version or version <= want_version: + keys.append((priority, version)) + + return max(keys) if keys else None + +def _python_tag_priority(*, tag, implementation, py_version): + if tag.startswith(PY_TAG_GENERIC): + ver_str = tag[len(PY_TAG_GENERIC):] + elif tag.startswith(implementation): + ver_str = tag[len(implementation):] + else: + return None + + # Add a 0 at the end in case it is a single digit + ver_str = "{}.{}".format(ver_str[0], ver_str[1:] or "0") + + ver = version.parse(ver_str) + if not version.is_compatible(py_version, ver): + return None + + return ( + tag.startswith(implementation), + version.key(ver), + ) + +def _candidates_by_priority( + *, + whls, + implementation_name, + python_version, + whl_abi_tags, + whl_platform_tags, + logger): + """Calculate the priority of each wheel + + Returns: + A dictionary where keys are priority tuples which allows us to sort and pick the + last item. + """ + py_version = version.parse(python_version, strict = True) + implementation = python_tag(implementation_name) + + ret = {} + for whl in whls: + parsed = parse_whl_name(whl.filename) + priority = None + + # See https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#compressed-tag-sets + for platform in parsed.platform_tag.split("."): + platform = _platform_tag_priority(tag = platform, values = whl_platform_tags) + if platform == None: + if logger: + logger.debug(lambda: "The platform_tag in '{}' does not match given list: {}".format( + whl.filename, + whl_platform_tags, + )) + continue + + for py in parsed.python_tag.split("."): + py = _python_tag_priority( + tag = py, + implementation = implementation, + py_version = py_version, + ) + if py == None: + if logger: + logger.debug(lambda: "The python_tag in '{}' does not match implementation or version: {} {}".format( + whl.filename, + implementation, + py_version.string, + )) + continue + + for abi in parsed.abi_tag.split("."): + abi = _value_priority( + tag = abi, + values = whl_abi_tags, + ) + if abi == None: + if logger: + logger.debug(lambda: "The abi_tag in '{}' does not match given list: {}".format( + whl.filename, + whl_abi_tags, + )) + continue + + # 1. Prefer platform wheels + # 2. Then prefer implementation/python version + # 3. Then prefer more specific ABI wheels + candidate = (platform, py, abi) + priority = priority or candidate + if candidate > priority: + priority = candidate + + if priority == None: + if logger: + logger.debug(lambda: "The whl '{}' is incompatible".format( + whl.filename, + )) + continue + + ret[priority] = whl + + return ret + +def select_whl( + *, + whls, + python_version, + whl_platform_tags, + whl_abi_tags, + implementation_name = "cpython", + limit = 1, + logger = None): + """Select a whl that is the most suitable for the given platform. + + Args: + whls: {type}`list[struct]` a list of candidates which have a `filename` + attribute containing the `whl` filename. + python_version: {type}`str` the target python version. + implementation_name: {type}`str` the `implementation_name` from the target_platform env. + whl_abi_tags: {type}`list[str]` The whl abi tags to select from. The preference is + for wheels that have ABI values appearing later in the `whl_abi_tags` list. + whl_platform_tags: {type}`list[str]` The whl platform tags to select from. + The platform tag may contain `*` and this means that if the platform tag is + versioned (e.g. `manylinux`), then we will select the highest available + platform version, e.g. if `manylinux_2_17` and `manylinux_2_5` wheels are both + compatible, we will select `manylinux_2_17`. Otherwise for versioned platform + tags we select the highest *compatible* version, e.g. if `manylinux_2_6` + support is requested, then we would select `manylinux_2_5` in the previous + example. This allows us to pass the same filtering parameters when selecting + all of the whl dependencies irrespective of what actual platform tags they + contain. + limit: {type}`int` number of wheels to return. Defaults to 1. + logger: {type}`struct` the logger instance. + + Returns: + {type}`list[struct] | struct | None`, a single struct from the `whls` input + argument or `None` if a match is not found. If the `limit` is greater than + one, then we will return a list. + """ + candidates = _candidates_by_priority( + whls = whls, + implementation_name = implementation_name, + python_version = python_version, + whl_abi_tags = whl_abi_tags, + whl_platform_tags = whl_platform_tags, + logger = logger, + ) + + if not candidates: + return None + + res = [i[1] for i in sorted(candidates.items())] + if logger: + logger.debug(lambda: "Sorted candidates:\n{}".format( + "\n".join([c.filename for c in res]), + )) + + return res[-1] if limit == 1 else res[-limit:] diff --git a/tests/pypi/select_whl/BUILD.bazel b/tests/pypi/select_whl/BUILD.bazel new file mode 100644 index 0000000000..0ad8cba0cd --- /dev/null +++ b/tests/pypi/select_whl/BUILD.bazel @@ -0,0 +1,3 @@ +load(":select_whl_tests.bzl", "select_whl_test_suite") + +select_whl_test_suite(name = "select_whl_tests") diff --git a/tests/pypi/select_whl/select_whl_tests.bzl b/tests/pypi/select_whl/select_whl_tests.bzl new file mode 100644 index 0000000000..28e17ba3b3 --- /dev/null +++ b/tests/pypi/select_whl/select_whl_tests.bzl @@ -0,0 +1,463 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "REPO_VERBOSITY_ENV_VAR", "repo_utils") # buildifier: disable=bzl-visibility +load("//python/private/pypi:select_whl.bzl", "select_whl") # buildifier: disable=bzl-visibility + +WHL_LIST = [ + "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", + "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp311-cp311-win32.whl", + "pkg-0.0.1-cp311-cp311-win_amd64.whl", + "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-win32.whl", + "pkg-0.0.1-cp37-cp37m-win_amd64.whl", + "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl", + "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp39-cp39-win32.whl", + "pkg-0.0.1-cp39-cp39-win_amd64.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-py310-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + # Extra examples that should be discarded + "pkg-0.0.1-py27-cp27mu-win_amd64.whl", + "pkg-0.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", +] + +def _match(env, got, *want_filenames): + if not want_filenames: + env.expect.that_collection(got).has_size(len(want_filenames)) + return + + got = [g for g in got if g] + got_filenames = [g.filename for g in got] + env.expect.that_collection(got_filenames).contains_exactly(want_filenames).in_order() + + if got: + # Check that we pass the original structs + env.expect.that_str(got[0].other).equals("dummy") + +def _select_whl(whls, debug = False, **kwargs): + return select_whl( + whls = [ + struct( + filename = f, + other = "dummy", + ) + for f in whls + ], + logger = repo_utils.logger(struct( + os = struct( + environ = { + REPO_DEBUG_ENV_VAR: "1", + REPO_VERBOSITY_ENV_VAR: "TRACE" if debug else "INFO", + }, + ), + ), "unit-test"), + **kwargs + ) + +_tests = [] + +def _test_not_select_py2(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py2-none-any.whl", + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + ], + whl_platform_tags = ["any"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 2, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + ) + +_tests.append(_test_not_select_py2) + +def _test_not_select_abi3(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-any.whl", + # the following should be ignored + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-p1.p2.p2.whl", + ], + whl_platform_tags = ["any", "p1"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 2, + debug = True, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + ) + +_tests.append(_test_not_select_abi3) + +def _test_select_cp312(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py2-none-any.whl", + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + "pkg-0.0.1-cp39-none-any.whl", + "pkg-0.0.1-cp312-none-any.whl", + "pkg-0.0.1-cp314-none-any.whl", + ], + whl_platform_tags = ["any"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 5, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + "pkg-0.0.1-cp39-none-any.whl", + "pkg-0.0.1-cp312-none-any.whl", + ) + +_tests.append(_test_select_cp312) + +def _test_simplest(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + ] + + got = _select_whl( + whls = whls, + whl_platform_tags = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.0", + ) + _match( + env, + [got], + "pkg-0.0.1-py3-abi3-any.whl", + ) + +_tests.append(_test_simplest) + +def _test_select_by_supported_py_version(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + ] + + for minor_version, match in { + 8: "pkg-0.0.1-py3-abi3-any.whl", + 11: "pkg-0.0.1-py311-abi3-any.whl", + }.items(): + got = _select_whl( + whls = whls, + whl_platform_tags = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.{}".format(minor_version), + ) + _match(env, [got], match) + +_tests.append(_test_select_by_supported_py_version) + +def _test_select_by_supported_cp_version(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + "pkg-0.0.1-cp311-abi3-any.whl", + ] + + for minor_version, match in { + 11: "pkg-0.0.1-cp311-abi3-any.whl", + 8: "pkg-0.0.1-py3-abi3-any.whl", + }.items(): + got = _select_whl( + whls = whls, + whl_platform_tags = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.{}".format(minor_version), + ) + _match(env, [got], match) + +_tests.append(_test_select_by_supported_cp_version) + +def _test_supported_cp_version_manylinux(env): + whls = [ + "pkg-0.0.1-py2.py3-none-manylinux_1_1_x86_64.whl", + "pkg-0.0.1-py3-none-manylinux_1_1_x86_64.whl", + "pkg-0.0.1-py311-none-manylinux_1_1_x86_64.whl", + "pkg-0.0.1-cp311-none-manylinux_1_1_x86_64.whl", + ] + + for minor_version, match in { + 8: "pkg-0.0.1-py3-none-manylinux_1_1_x86_64.whl", + 11: "pkg-0.0.1-cp311-none-manylinux_1_1_x86_64.whl", + }.items(): + got = _select_whl( + whls = whls, + whl_platform_tags = ["manylinux_1_1_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.{}".format(minor_version), + ) + _match(env, [got], match) + +_tests.append(_test_supported_cp_version_manylinux) + +def _test_ignore_unsupported(env): + whls = ["pkg-0.0.1-xx3-abi3-any.whl"] + got = _select_whl( + whls = whls, + whl_platform_tags = ["any"], + whl_abi_tags = ["none"], + python_version = "3.0", + ) + if got: + _match(env, [got], None) + +_tests.append(_test_ignore_unsupported) + +def _test_match_abi_and_not_py_version(env): + # Check we match the ABI and not the py version + whls = WHL_LIST + whl_platform_tags = [ + "musllinux_*_x86_64", + "manylinux_*_x86_64", + ] + got = _select_whl( + whls = whls, + whl_platform_tags = whl_platform_tags, + whl_abi_tags = ["abi3", "cp37m"], + python_version = "3.7", + ) + _match( + env, + [got], + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + ) + + got = _select_whl( + whls = whls, + whl_platform_tags = whl_platform_tags[::-1], + whl_abi_tags = ["abi3", "cp37m"], + python_version = "3.7", + ) + _match( + env, + [got], + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_match_abi_and_not_py_version) + +def _test_select_filename_with_many_tags(env): + # Check we can select a filename with many platform tags + got = _select_whl( + whls = WHL_LIST, + whl_platform_tags = [ + "any", + "musllinux_*_i686", + "manylinux_*_i686", + ], + whl_abi_tags = ["none", "abi3", "cp39"], + python_version = "3.9", + limit = 5, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + ) + +_tests.append(_test_select_filename_with_many_tags) + +def _test_freethreaded_wheels(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = WHL_LIST, + whl_platform_tags = [ + "any", + "musllinux_*_x86_64", + ], + whl_abi_tags = ["none", "abi3", "cp313", "cp313t"], + python_version = "3.13", + limit = 8, + ) + _match( + env, + got, + # The last item has the most priority + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py310-abi3-any.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_freethreaded_wheels) + +def _test_pytags_all_possible(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313-none-win_amd64.whl", + ], + whl_platform_tags = ["win_amd64"], + whl_abi_tags = ["none"], + python_version = "3.12", + ) + _match( + env, + [got], + "pkg-0.0.1-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313-none-win_amd64.whl", + ) + +_tests.append(_test_pytags_all_possible) + +def _test_manylinx_musllinux_pref(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-manylinux_2_31_x86_64.musllinux_1_1_x86_64.whl", + ], + whl_platform_tags = [ + "manylinux_*_x86_64", + "musllinux_*_x86_64", + ], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # there is only one wheel, just select that + "pkg-0.0.1-py3-none-manylinux_2_31_x86_64.musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_manylinx_musllinux_pref) + +def _test_multiple_musllinux(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ], + whl_platform_tags = ["musllinux_*_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # select the one with the highest version that is matching + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + ) + +_tests.append(_test_multiple_musllinux) + +def _test_multiple_musllinux_exact_params(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ], + whl_platform_tags = ["musllinux_1_2_x86_64", "musllinux_1_1_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # select the one with the lowest version, because of the input to the function + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_multiple_musllinux_exact_params) + +def _test_android(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-android_4_x86_64.whl", + "pkg-0.0.1-py3-none-android_8_x86_64.whl", + ], + whl_platform_tags = ["android_5_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # select the one with the highest version that is matching + "pkg-0.0.1-py3-none-android_4_x86_64.whl", + ) + +_tests.append(_test_android) + +def select_whl_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From afa76862fb6e27fc20920d8d2be1a906ebfb2fba Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 21 Jul 2025 08:00:04 +0900 Subject: [PATCH 3/6] fix(pypi): reuse select dicts for constructing the env Before this PR we would be constructing slightly different environments when the `env_marker_setting` is doing it in the analysis phase and when we are doing it in the repo phase due to how the defaults are handled. In this change we simply reuse the same select statements and add an extra helper that is allowing us to process that. Work towards #2949 Prep for #3058 --- python/private/pypi/BUILD.bazel | 8 +-- python/private/pypi/extension.bzl | 7 +- python/private/pypi/pep508_env.bzl | 86 +++++++++++++++---------- python/private/pypi/pep508_platform.bzl | 57 ---------------- tests/pypi/pep508/BUILD.bazel | 5 ++ tests/pypi/pep508/env_tests.bzl | 69 ++++++++++++++++++++ tests/pypi/pep508/evaluate_tests.bzl | 22 +++++-- 7 files changed, 150 insertions(+), 104 deletions(-) delete mode 100644 python/private/pypi/pep508_platform.bzl create mode 100644 tests/pypi/pep508/env_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f854d4edcc..4b56b73284 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -252,6 +252,9 @@ bzl_library( bzl_library( name = "pep508_env_bzl", srcs = ["pep508_env.bzl"], + deps = [ + "//python/private:version_bzl", + ], ) bzl_library( @@ -263,11 +266,6 @@ bzl_library( ], ) -bzl_library( - name = "pep508_platform_bzl", - srcs = ["pep508_platform.bzl"], -) - bzl_library( name = "pep508_requirement_bzl", srcs = ["pep508_requirement.bzl"], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 2c1528e18d..4fb95e06bb 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -76,11 +76,12 @@ def _platforms(*, python_version, minor_mapping, config): for platform, values in config.platforms.items(): key = "{}_{}".format(abi, platform) - platforms[key] = env(struct( - abi = abi, + platforms[key] = env( + env = values.env, os = values.os_name, arch = values.arch_name, - )) | values.env + python_version = python_version, + ) return platforms def _create_whl_repos( diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index c2d404bc3e..5031ebae12 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -15,6 +15,20 @@ """This module is for implementing PEP508 environment definition. """ +load("//python/private:version.bzl", "version") + +_DEFAULT = "//conditions:default" + +# Here we store the aliases in the platform so that the users can specify any valid target in +# there. +_cpu_aliases = { + "arm": "aarch32", + "arm64": "aarch64", +} +_os_aliases = { + "macos": "osx", +} + # See https://stackoverflow.com/a/45125525 platform_machine_aliases = { # These pairs mean the same hardware, but different values may be used @@ -59,7 +73,7 @@ platform_machine_select_map = { "@platforms//cpu:x86_64": "x86_64", # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine - "//conditions:default": "", + _DEFAULT: "", } # Platform system returns results from the `uname` call. @@ -73,7 +87,7 @@ _platform_system_values = { "linux": "Linux", "netbsd": "NetBSD", "openbsd": "OpenBSD", - "osx": "Darwin", + "osx": "Darwin", # NOTE: macos is an alias to osx, we handle it through _os_aliases "windows": "Windows", } @@ -83,7 +97,7 @@ platform_system_select_map = { } | { # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine - "//conditions:default": "", + _DEFAULT: "", } # The copy of SO [answer](https://stackoverflow.com/a/13874620) containing @@ -123,18 +137,19 @@ _sys_platform_values = { "ios": "ios", "linux": "linux", "openbsd": "openbsd", - "osx": "darwin", + "osx": "darwin", # NOTE: macos is an alias to osx, we handle it through _os_aliases "wasi": "wasi", "windows": "win32", } sys_platform_select_map = { + # These values are decided by the sys.platform docs. "@platforms//os:{}".format(bazel_os): py_platform for bazel_os, py_platform in _sys_platform_values.items() } | { # For lack of a better option, use empty string. No standard doc/spec # about sys_platform value. - "//conditions:default": "", + _DEFAULT: "", } # The "java" value is documented, but with Jython defunct, @@ -142,53 +157,58 @@ sys_platform_select_map = { # The os.name value is technically a property of the runtime, not the # targetted runtime OS, but the distinction shouldn't matter if # things are properly configured. -_os_name_values = { - "linux": "posix", - "osx": "posix", - "windows": "nt", -} - os_name_select_map = { - "@platforms//os:{}".format(bazel_os): py_os - for bazel_os, py_os in _os_name_values.items() -} | { - "//conditions:default": "posix", + "@platforms//os:windows": "nt", + _DEFAULT: "posix", } -def env(target_platform, *, extra = None): +def _set_default(env, env_key, m, key): + """Set the default value in the env if it is not already set.""" + default = m.get(key, m[_DEFAULT]) + env.setdefault(env_key, default) + +def env(*, env = None, os, arch, python_version = "", extra = None): """Return an env target platform NOTE: This is for use during the loading phase. For the analysis phase, `env_marker_setting()` constructs the env dict. Args: - target_platform: {type}`str` the target platform identifier, e.g. - `cp33_linux_aarch64` + env: {type}`str` the environment. + os: {type}`str` the OS name. + arch: {type}`str` the CPU name. + python_version: {type}`str` the full python version. extra: {type}`str` the extra value to be added into the env. Returns: A dict that can be used as `env` in the marker evaluation. """ - env = create_env() + env = env or {} + env = env | create_env() if extra != None: env["extra"] = extra - if target_platform.abi: - minor_version, _, micro_version = target_platform.abi[3:].partition(".") - micro_version = micro_version or "0" - env = env | { - "implementation_version": "3.{}.{}".format(minor_version, micro_version), - "python_full_version": "3.{}.{}".format(minor_version, micro_version), - "python_version": "3.{}".format(minor_version), - } - if target_platform.os and target_platform.arch: - os = target_platform.os + if python_version: + v = version.parse(python_version) + major = v.release[0] + minor = v.release[1] + micro = v.release[2] if len(v.release) > 2 else 0 env = env | { - "os_name": _os_name_values.get(os, ""), - "platform_machine": target_platform.arch, - "platform_system": _platform_system_values.get(os, ""), - "sys_platform": _sys_platform_values.get(os, ""), + "implementation_version": "{}.{}.{}".format(major, minor, micro), + "python_full_version": "{}.{}.{}".format(major, minor, micro), + "python_version": "{}.{}".format(major, minor), } + + if os: + os = "@platforms//os:{}".format(_os_aliases.get(os, os)) + _set_default(env, "os_name", os_name_select_map, os) + _set_default(env, "platform_system", platform_system_select_map, os) + _set_default(env, "sys_platform", sys_platform_select_map, os) + + if arch: + arch = "@platforms//cpu:{}".format(_cpu_aliases.get(arch, arch)) + _set_default(env, "platform_machine", platform_machine_select_map, arch) + set_missing_env_defaults(env) return env diff --git a/python/private/pypi/pep508_platform.bzl b/python/private/pypi/pep508_platform.bzl deleted file mode 100644 index 381a8d7a08..0000000000 --- a/python/private/pypi/pep508_platform.bzl +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2025 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""The platform abstraction -""" - -def platform(*, abi = None, os = None, arch = None): - """platform returns a struct for the platform. - - Args: - abi: {type}`str | None` the target ABI, e.g. `"cp39"`. - os: {type}`str | None` the target os, e.g. `"linux"`. - arch: {type}`str | None` the target CPU, e.g. `"aarch64"`. - - Returns: - A struct. - """ - - # Note, this is used a lot as a key in dictionaries, so it cannot contain - # methods. - return struct( - abi = abi, - os = os, - arch = arch, - ) - -def platform_from_str(p, python_version): - """Return a platform from a string. - - Args: - p: {type}`str` the actual string. - python_version: {type}`str` the python version to add to platform if needed. - - Returns: - A struct that is returned by the `_platform` function. - """ - if p.startswith("cp"): - abi, _, p = p.partition("_") - elif python_version: - major, _, tail = python_version.partition(".") - abi = "cp{}{}".format(major, tail) - else: - abi = None - - os, _, arch = p.partition("_") - return platform(abi = abi, os = os or None, arch = arch or None) diff --git a/tests/pypi/pep508/BUILD.bazel b/tests/pypi/pep508/BUILD.bazel index 7eab2e096a..36fce0fa89 100644 --- a/tests/pypi/pep508/BUILD.bazel +++ b/tests/pypi/pep508/BUILD.bazel @@ -1,4 +1,5 @@ load(":deps_tests.bzl", "deps_test_suite") +load(":env_tests.bzl", "env_test_suite") load(":evaluate_tests.bzl", "evaluate_test_suite") load(":requirement_tests.bzl", "requirement_test_suite") @@ -6,6 +7,10 @@ deps_test_suite( name = "deps_tests", ) +env_test_suite( + name = "env_tests", +) + evaluate_test_suite( name = "evaluate_tests", ) diff --git a/tests/pypi/pep508/env_tests.bzl b/tests/pypi/pep508/env_tests.bzl new file mode 100644 index 0000000000..cfd94a1b01 --- /dev/null +++ b/tests/pypi/pep508/env_tests.bzl @@ -0,0 +1,69 @@ +"""Tests to check for env construction.""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_env_defaults(env): + got = pep508_env(os = "exotic", arch = "exotic", python_version = "3.1.1") + got.pop("_aliases") + env.expect.that_dict(got).contains_exactly({ + "implementation_name": "cpython", + "implementation_version": "3.1.1", + "os_name": "posix", + "platform_machine": "", + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": "", + "platform_version": "0", + "python_full_version": "3.1.1", + "python_version": "3.1", + "sys_platform": "", + }) + +_tests.append(_test_env_defaults) + +def _test_env_freebsd(env): + got = pep508_env(os = "freebsd", arch = "arm64", python_version = "3.1.1") + got.pop("_aliases") + env.expect.that_dict(got).contains_exactly({ + "implementation_name": "cpython", + "implementation_version": "3.1.1", + "os_name": "posix", + "platform_machine": "aarch64", + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": "FreeBSD", + "platform_version": "0", + "python_full_version": "3.1.1", + "python_version": "3.1", + "sys_platform": "freebsd", + }) + +_tests.append(_test_env_freebsd) + +def _test_env_macos(env): + got = pep508_env(os = "macos", arch = "arm64", python_version = "3.1.1") + got.pop("_aliases") + env.expect.that_dict(got).contains_exactly({ + "implementation_name": "cpython", + "implementation_version": "3.1.1", + "os_name": "posix", + "platform_machine": "aarch64", + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": "Darwin", + "platform_version": "0", + "python_full_version": "3.1.1", + "python_version": "3.1", + "sys_platform": "darwin", + }) + +_tests.append(_test_env_macos) + +def env_test_suite(name): # buildifier: disable=function-docstring + test_suite( + name = name, + basic_tests = _tests, + ) diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index cc867f346c..7843f88e89 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -16,7 +16,6 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility load("//python/private/pypi:pep508_evaluate.bzl", "evaluate", "tokenize") # buildifier: disable=bzl-visibility -load("//python/private/pypi:pep508_platform.bzl", "platform_from_str") # buildifier: disable=bzl-visibility _tests = [] @@ -244,26 +243,37 @@ _tests.append(_evaluate_partial_only_extra) def _evaluate_with_aliases(env): # When - for target_platform, tests in { + for (os, cpu), tests in { # buildifier: @unsorted-dict-items - "osx_aarch64": { + ("osx", "aarch64"): { "platform_system == 'Darwin' and platform_machine == 'arm64'": True, "platform_system == 'Darwin' and platform_machine == 'aarch64'": True, "platform_system == 'Darwin' and platform_machine == 'amd64'": False, }, - "osx_x86_64": { + ("osx", "x86_64"): { "platform_system == 'Darwin' and platform_machine == 'amd64'": True, "platform_system == 'Darwin' and platform_machine == 'x86_64'": True, }, - "osx_x86_32": { + ("osx", "x86_32"): { "platform_system == 'Darwin' and platform_machine == 'i386'": True, "platform_system == 'Darwin' and platform_machine == 'i686'": True, "platform_system == 'Darwin' and platform_machine == 'x86_32'": True, "platform_system == 'Darwin' and platform_machine == 'x86_64'": False, }, + ("freebsd", "x86_32"): { + "platform_system == 'FreeBSD' and platform_machine == 'i386'": True, + "platform_system == 'FreeBSD' and platform_machine == 'i686'": True, + "platform_system == 'FreeBSD' and platform_machine == 'x86_32'": True, + "platform_system == 'FreeBSD' and platform_machine == 'x86_64'": False, + "platform_system == 'FreeBSD' and os_name == 'posix'": True, + }, }.items(): # buildifier: @unsorted-dict-items for input, want in tests.items(): - _check_evaluate(env, input, want, pep508_env(platform_from_str(target_platform, ""))) + _check_evaluate(env, input, want, pep508_env( + os = os, + arch = cpu, + python_version = "3.2", + )) _tests.append(_evaluate_with_aliases) From f2e68cf9302781af7fe86e936f46987150240e39 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 21 Jul 2025 12:24:53 +0900 Subject: [PATCH 4/6] feat(pypi): incrementally build platform configuration Before this PR the configuration for platforms would be built non-incrementally, making it harder for users to override particular attributes of the already configured ones. With this PR the new features introduced in #3058 will be easier to override. Work towards #2747 --- python/private/pypi/extension.bzl | 111 ++++++++++++++++------- tests/pypi/extension/extension_tests.bzl | 53 ++++++++++- 2 files changed, 128 insertions(+), 36 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 4fb95e06bb..22aac8c5f0 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -380,7 +380,7 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) - if platform: + if platform and (os_name or arch_name or config_settings or env): if not override and config.get("platforms", {}).get(platform): return @@ -388,16 +388,82 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { if key not in _SUPPORTED_PEP508_KEYS: fail("Unsupported key in the PEP508 environment: {}".format(key)) - config["platforms"][platform] = struct( - name = platform.replace("-", "_").lower(), - os_name = os_name, - arch_name = arch_name, - config_settings = config_settings, - env = env, - ) + config["platforms"].setdefault(platform, {}) + for key, value in { + "arch_name": arch_name, + "config_settings": config_settings, + "env": env, + "name": platform.replace("-", "_").lower(), + "os_name": os_name, + }.items(): + if not value: + continue + + if not override and config.get(key): + continue + + config["platforms"][platform][key] = value else: config["platforms"].pop(platform) +def _plat(*, name, arch_name, os_name, config_settings = [], env = {}): + return struct( + name = name, + arch_name = arch_name, + os_name = os_name, + config_settings = config_settings, + env = env, + ) + +def build_config( + *, + module_ctx, + enable_pipstar): + """Parse 'configure' and 'default' extension tags + + Args: + module_ctx: {type}`module_ctx` module context. + enable_pipstar: {type}`bool` a flag to enable dropping Python dependency for + evaluation of the extension. + + Returns: + A struct with the configuration. + """ + defaults = { + "platforms": {}, + } + for mod in module_ctx.modules: + if not (mod.is_root or mod.name == "rules_python"): + continue + + platform = None + for tag in mod.tags.default: + platform = tag.platform or platform + _configure( + defaults, + arch_name = tag.arch_name, + config_settings = tag.config_settings, + env = tag.env, + os_name = tag.os_name, + platform = platform, + override = mod.is_root, + # TODO @aignas 2025-05-19: add more attr groups: + # * for AUTH - the default `netrc` usage could be configured through a common + # attribute. + # * for index/downloader config. This includes all of those attributes for + # overrides, etc. Index overrides per platform could be also used here. + # * for whl selection - selecting preferences of which `platform_tag`s we should use + # for what. We could also model the `cp313t` freethreaded as separate platforms. + ) + + return struct( + platforms = { + name: _plat(**values) + for name, values in defaults["platforms"].items() + }, + enable_pipstar = enable_pipstar, + ) + def parse_modules( module_ctx, _fail = fail, @@ -448,33 +514,7 @@ You cannot use both the additive_build_content and additive_build_content_file a srcs_exclude_glob = whl_mod.srcs_exclude_glob, ) - defaults = { - "enable_pipstar": enable_pipstar, - "platforms": {}, - } - for mod in module_ctx.modules: - if not (mod.is_root or mod.name == "rules_python"): - continue - - for tag in mod.tags.default: - _configure( - defaults, - arch_name = tag.arch_name, - config_settings = tag.config_settings, - env = tag.env, - os_name = tag.os_name, - platform = tag.platform, - override = mod.is_root, - # TODO @aignas 2025-05-19: add more attr groups: - # * for AUTH - the default `netrc` usage could be configured through a common - # attribute. - # * for index/downloader config. This includes all of those attributes for - # overrides, etc. Index overrides per platform could be also used here. - # * for whl selection - selecting preferences of which `platform_tag`s we should use - # for what. We could also model the `cp313t` freethreaded as separate platforms. - ) - - config = struct(**defaults) + config = build_config(module_ctx = module_ctx, enable_pipstar = enable_pipstar) # TODO @aignas 2025-06-03: Merge override API with the builder? _overriden_whl_set = {} @@ -651,6 +691,7 @@ You cannot use both the additive_build_content and additive_build_content_file a k: dict(sorted(args.items())) for k, args in sorted(whl_libraries.items()) }, + config = config, ) def _pip_impl(module_ctx): diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 0303843e80..04affd3adf 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -16,7 +16,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") -load("//python/private/pypi:extension.bzl", "parse_modules") # buildifier: disable=bzl-visibility +load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # buildifier: disable=bzl-visibility load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility @@ -92,6 +92,18 @@ def _parse_modules(env, enable_pipstar = 0, **kwargs): ), ) +def _build_config(env, enable_pipstar = 0, **kwargs): + return env.expect.that_struct( + build_config( + enable_pipstar = enable_pipstar, + **kwargs + ), + attrs = dict( + platforms = subjects.dict, + enable_pipstar = subjects.bool, + ), + ) + def _default( arch_name = None, config_settings = None, @@ -1108,6 +1120,45 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' _tests.append(_test_pipstar_platforms) +def _test_build_pipstar_platform(env): + config = _build_config( + env, + module_ctx = _mock_mctx( + _mod( + name = "rules_python", + default = [ + _default( + platform = "myplat", + os_name = "linux", + arch_name = "x86_64", + ), + _default( + config_settings = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + ], + ), + ), + enable_pipstar = True, + ) + config.enable_pipstar().equals(True) + config.platforms().contains_exactly({ + "myplat": struct( + name = "myplat", + os_name = "linux", + arch_name = "x86_64", + config_settings = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + env = {}, + ), + }) + +_tests.append(_test_build_pipstar_platform) + def extension_test_suite(name): """Create the test suite. From 89e58de8bf12627e885b2e2ef13021e33101535c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:40:57 +0900 Subject: [PATCH 5/6] fix(pypi): pull fewer wheels with experimental_index_url Before this we would pull all of the wheels that the user target configuration would be compatible with and that meant that it was not customizable. This also meant that there were a lot of footguns in the configuration where the select statements were not really foolproof. With this PR we select only those sources that need to be for the declared configurations. Freethreaded support is done by defining extra freethreaded platforms using the new builder API (#3063). This is also changing the default platforms to be only the fully supported platforms. This makes the testing easier and avoids us running into compatibility issues during the roll out. Work towards #2747 Fixes #2759 Fixes #2849 --- CHANGELOG.md | 8 + MODULE.bazel | 69 ++-- examples/bzlmod/MODULE.bazel | 17 + python/private/pypi/BUILD.bazel | 5 +- python/private/pypi/evaluate_markers.bzl | 6 +- python/private/pypi/extension.bzl | 259 +++++++++++---- python/private/pypi/parse_requirements.bzl | 94 +++--- python/private/pypi/whl_target_platforms.bzl | 132 -------- tests/pypi/extension/extension_tests.bzl | 157 ++++----- .../parse_requirements_tests.bzl | 33 +- tests/pypi/whl_target_platforms/BUILD.bazel | 3 - .../whl_target_platforms/select_whl_tests.bzl | 314 ------------------ 12 files changed, 425 insertions(+), 672 deletions(-) delete mode 100644 tests/pypi/whl_target_platforms/select_whl_tests.bzl diff --git a/CHANGELOG.md b/CHANGELOG.md index b36ceafa27..9258f74fec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,10 @@ END_UNRELEASED_TEMPLATE * (toolchain) Python 3.13 now references 3.13.5 * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) +* (pypi) From now on the list of default platforms only includes `linux_x86_64`, `linux_aarch64`, + `osx_x86_64`, `osx_aarch64` and `windows_x86_64`. If you are on other platforms, you need to + use the `pip.default` to configure it yourself. If you are interested in graduating the + platform, consider helping set us up CI for them and update the documentation. * (ci) We are now testing on Ubuntu 22.04 for RBE and non-RBE configurations. * (core) #!/usr/bin/env bash is now used as a shebang in the stage1 bootstrap template. @@ -88,6 +92,10 @@ END_UNRELEASED_TEMPLATE ([#3043](https://github.com/bazel-contrib/rules_python/issues/3043)). * (pypi) The pipstar `defaults` configuration now supports any custom platform name. +* (pypi) The selection of the whls has been changed and should no longer result + in ambiguous select matches ({gh-issue}`2759`) and should be much more efficient + when running `bazel query` due to fewer repositories being included + ({gh-issue}`2849`). * Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle. * (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8 ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)). diff --git a/MODULE.bazel b/MODULE.bazel index 9db287dc28..97de1ad6eb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -74,16 +74,18 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") env = {"platform_version": "0"}, os_name = "linux", platform = "linux_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = [ + "linux_{}".format(cpu), + "manylinux_*_{}".format(cpu), + ], ) for cpu in [ "x86_64", "aarch64", - # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the - # `pip.default` extension. i.e. drop the below values - users will have to - # define themselves if they need them. - "arm", - "ppc", - "s390x", ] ] @@ -99,26 +101,53 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") env = {"platform_version": "14.0"}, os_name = "osx", platform = "osx_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = [ + "macosx_*_{}".format(suffix) + for suffix in platform_tag_cpus + ], ) - for cpu in [ - "aarch64", - "x86_64", - ] + for cpu, platform_tag_cpus in { + "aarch64": [ + "universal2", + "arm64", + ], + "x86_64": [ + "universal2", + "x86_64", + ], + }.items() +] + +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:windows", + ], + env = {"platform_version": "0"}, + os_name = "windows", + platform = "windows_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = whl_platform_tags, + ) + for cpu, whl_platform_tags in { + "x86_64": ["win_amd64"], + }.items() ] -pip.default( - arch_name = "x86_64", - config_settings = [ - "@platforms//cpu:x86_64", - "@platforms//os:windows", - ], - env = {"platform_version": "0"}, - os_name = "windows", - platform = "windows_x86_64", -) pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us # being able to build sdists for this hub, so explicitly set this to False. + # + # how do we test sdists? Maybe just worth adding a single sdist somewhere? download_only = False, experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 841c096dcf..95e1090f53 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -158,6 +158,23 @@ pip.whl_mods( ) use_repo(pip, "whl_mods_hub") +# Because below we are using `windows_aarch64` platform, we have to define various +# properties for it. +pip.default( + arch_name = "aarch64", + config_settings = [ + "@platforms//os:windows", + "@platforms//cpu:aarch64", + ], + env = { + "platform_version": "0", + }, + os_name = "windows", + platform = "windows_aarch64", + whl_abi_tags = [], # default to all ABIs + whl_platform_tags = ["win_amd64"], +) + # To fetch pip dependencies, use pip.parse. We can pass in various options, # but typically we pass requirements and the Python version. The Python # version must have been configured by a corresponding `python.toolchain()` diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 4b56b73284..847c85d634 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -116,11 +116,11 @@ bzl_library( ":parse_whl_name_bzl", ":pep508_env_bzl", ":pip_repository_attrs_bzl", + ":python_tag_bzl", ":simpleapi_download_bzl", ":whl_config_setting_bzl", ":whl_library_bzl", ":whl_repo_name_bzl", - ":whl_target_platforms_bzl", "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", "//python/private:version_bzl", @@ -209,7 +209,7 @@ bzl_library( ":parse_requirements_txt_bzl", ":pypi_repo_utils_bzl", ":requirements_files_by_platform_bzl", - ":whl_target_platforms_bzl", + ":select_whl_bzl", "//python/private:normalize_name_bzl", "//python/private:repo_utils_bzl", ], @@ -438,5 +438,4 @@ bzl_library( bzl_library( name = "whl_target_platforms_bzl", srcs = ["whl_target_platforms.bzl"], - deps = [":parse_whl_name_bzl"], ) diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 6167cdbc96..4d6a39a1df 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -43,11 +43,11 @@ def evaluate_markers(*, requirements, platforms): for req_string, platform_strings in requirements.items(): req = requirement(req_string) for platform_str in platform_strings: - env = platforms.get(platform_str) - if not env: + plat = platforms.get(platform_str) + if not plat: fail("Please define platform: '{}'".format(platform_str)) - if evaluate(req.marker, env = env): + if evaluate(req.marker, env = plat.env): ret.setdefault(req_string, []).append(platform_str) return ret diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 22aac8c5f0..c16c3c6de0 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -31,6 +31,7 @@ load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") load(":pip_repository_attrs.bzl", "ATTRS") +load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") load(":whl_config_setting.bzl", "whl_config_setting") @@ -68,19 +69,40 @@ def _whl_mods_impl(whl_mods_dict): def _platforms(*, python_version, minor_mapping, config): platforms = {} - python_version = full_version( - version = python_version, - minor_mapping = minor_mapping, + python_version = version.parse( + full_version( + version = python_version, + minor_mapping = minor_mapping, + ), + strict = True, ) - abi = "cp3{}".format(python_version[2:]) for platform, values in config.platforms.items(): - key = "{}_{}".format(abi, platform) - platforms[key] = env( - env = values.env, - os = values.os_name, - arch = values.arch_name, - python_version = python_version, + # TODO @aignas 2025-07-07: this is probably doing the parsing of the version too + # many times. + key = "{}{}{}.{}_{}".format( + python_tag(values.env["implementation_name"]), + python_version.release[0], + python_version.release[1], + python_version.release[2], + platform, + ) + + platforms[key] = struct( + env = env( + env = values.env, + os = values.os_name, + arch = values.arch_name, + python_version = python_version.string, + ), + whl_abi_tags = [ + v.format( + major = python_version.release[0], + minor = python_version.release[1], + ) + for v in values.whl_abi_tags + ], + whl_platform_tags = values.whl_platform_tags, ) return platforms @@ -153,6 +175,8 @@ def _create_whl_repos( )) python_interpreter_target = available_interpreters[python_name] + # TODO @aignas 2025-06-29: we should not need the version in the pip_name if + # we are using pipstar and we are downloading the wheel using the downloader pip_name = "{}_{}".format( hub_name, version_label(pip_attr.python_version), @@ -231,12 +255,21 @@ def _create_whl_repos( ), logger = logger, ), + platforms = _platforms( + python_version = pip_attr.python_version, + minor_mapping = minor_mapping, + config = config, + ), extra_pip_args = pip_attr.extra_pip_args, get_index_urls = get_index_urls, evaluate_markers = evaluate_markers, logger = logger, ) + use_downloader = { + normalize_name(s): False + for s in pip_attr.simpleapi_skip + } exposed_packages = {} for whl in requirements_by_platform: if whl.is_exposed: @@ -290,11 +323,20 @@ def _create_whl_repos( whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = pip_attr.netrc, + use_downloader = use_downloader.get( + whl.name, + get_index_urls != None, # defaults to True if the get_index_urls is defined + ), auth_patterns = pip_attr.auth_patterns, python_version = major_minor, is_multiple_versions = whl.is_multiple_versions, enable_pipstar = config.enable_pipstar, ) + if repo == None: + # NOTE @aignas 2025-07-07: we guard against an edge-case where there + # are more platforms defined than there are wheels for and users + # disallow building from sdist. + continue repo_name = "{}_{}".format(pip_name, repo.repo_name) if repo_name in whl_libraries: @@ -313,7 +355,17 @@ def _create_whl_repos( whl_libraries = whl_libraries, ) -def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, netrc, auth_patterns, python_version, enable_pipstar = False): +def _whl_repo( + *, + src, + whl_library_args, + is_multiple_versions, + download_only, + netrc, + auth_patterns, + python_version, + use_downloader, + enable_pipstar = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -326,19 +378,24 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net args["extra_pip_args"] = src.extra_pip_args if not src.url or (not is_whl and download_only): - # Fallback to a pip-installed wheel - target_platforms = src.target_platforms if is_multiple_versions else [] - return struct( - repo_name = pypi_repo_name( - normalize_name(src.distribution), - *target_platforms - ), - args = args, - config_setting = whl_config_setting( - version = python_version, - target_platforms = target_platforms or None, - ), - ) + if download_only and use_downloader: + # If the user did not allow using sdists and we are using the downloader + # and we are not using simpleapi_skip for this + return None + else: + # Fallback to a pip-installed wheel + target_platforms = src.target_platforms if is_multiple_versions else [] + return struct( + repo_name = pypi_repo_name( + normalize_name(src.distribution), + *target_platforms + ), + args = args, + config_setting = whl_config_setting( + version = python_version, + target_platforms = target_platforms or None, + ), + ) # This is no-op because pip is not used to download the wheel. args.pop("download_only", None) @@ -360,28 +417,33 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net for p in src.target_platforms ] - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if is_whl and not src.filename.endswith("-any.whl"): - pass - elif is_multiple_versions: - target_platforms = src.target_platforms - return struct( repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, - filename = src.filename, - target_platforms = target_platforms, + target_platforms = src.target_platforms, ), ) -def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False): +def _configure( + config, + *, + platform, + os_name, + arch_name, + config_settings, + env = {}, + whl_abi_tags, + whl_platform_tags, + override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) - if platform and (os_name or arch_name or config_settings or env): - if not override and config.get("platforms", {}).get(platform): + + if platform and ( + os_name or arch_name or config_settings or whl_abi_tags or whl_platform_tags or env + ): + if not override and config["platforms"].get(platform): return for key in env: @@ -395,6 +457,8 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { "env": env, "name": platform.replace("-", "_").lower(), "os_name": os_name, + "whl_abi_tags": whl_abi_tags, + "whl_platform_tags": whl_platform_tags, }.items(): if not value: continue @@ -406,13 +470,28 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { else: config["platforms"].pop(platform) -def _plat(*, name, arch_name, os_name, config_settings = [], env = {}): +def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, whl_abi_tags = [], whl_platform_tags = []): + # NOTE @aignas 2025-07-08: the least preferred is the first item in the list + if "any" not in whl_platform_tags: + # the lowest priority one needs to be the first one + whl_platform_tags = ["any"] + whl_platform_tags + + whl_abi_tags = whl_abi_tags or ["abi3", "cp{major}{minor}"] + if "none" not in whl_abi_tags: + # the lowest priority one needs to be the first one + whl_abi_tags = ["none"] + whl_abi_tags + return struct( name = name, arch_name = arch_name, os_name = os_name, config_settings = config_settings, - env = env, + env = { + # defaults for env + "implementation_name": "cpython", + } | env, + whl_abi_tags = whl_abi_tags, + whl_platform_tags = whl_platform_tags, ) def build_config( @@ -447,6 +526,8 @@ def build_config( os_name = tag.os_name, platform = platform, override = mod.is_root, + whl_abi_tags = tag.whl_abi_tags, + whl_platform_tags = tag.whl_platform_tags, # TODO @aignas 2025-05-19: add more attr groups: # * for AUTH - the default `netrc` usage could be configured through a common # attribute. @@ -643,7 +724,14 @@ You cannot use both the additive_build_content and additive_build_content_file a for whl_name, aliases in out.extra_aliases.items(): extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases) exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages) - whl_libraries.update(out.whl_libraries) + for whl_name, lib in out.whl_libraries.items(): + if enable_pipstar: + whl_libraries.setdefault(whl_name, lib) + elif whl_name in lib: + fail("'{}' already in created".format(whl_name)) + else: + # replicate whl_libraries.update(out.whl_libraries) + whl_libraries[whl_name] = lib # TODO @aignas 2024-04-05: how do we support different requirement # cycles for different abis/oses? For now we will need the users to @@ -794,6 +882,7 @@ _default_attrs = { "arch_name": attr.string( doc = """\ The CPU architecture name to be used. +You can use any cpu name from the `@platforms//cpu:` package. :::{note} Either this or {attr}`env` `platform_machine` key should be specified. @@ -807,25 +896,6 @@ The list of labels to `config_setting` targets that need to be matched for the p selected. """, ), - "os_name": attr.string( - doc = """\ -The OS name to be used. - -:::{note} -Either this or the appropriate `env` keys should be specified. -::: -""", - ), - "platform": attr.string( - doc = """\ -A platform identifier which will be used as the unique identifier within the extension evaluation. -If you are defining custom platforms in your project and don't want things to clash, use extension -[isolation] feature. - -[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate -""", - ), -} | { "env": attr.string_dict( doc = """\ The values to use for environment markers when evaluating an expression. @@ -851,6 +921,79 @@ This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled. """, ), # The values for PEP508 env marker evaluation during the lock file parsing + "os_name": attr.string( + doc = """\ +The OS name to be used. +You can use any OS name from the `@platforms//os:` package. + +:::{note} +Either this or the appropriate `env` keys should be specified. +::: +""", + ), + "platform": attr.string( + doc = """\ +A platform identifier which will be used as the unique identifier within the extension evaluation. +If you are defining custom platforms in your project and don't want things to clash, use extension +[isolation] feature. + +[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate +""", + ), + "whl_abi_tags": attr.string_list( + doc = """\ +A list of ABIs to select wheels for. The values can be either strings or include template +parameters like `{major}` and `{minor}` which will be replaced with python version parts. e.g. +`cp{major}{minor}` will result in `cp313` given the full python version is `3.13.5`. +Will always include `"none"` even if it is not specified. + +:::{note} +We select a single wheel and the last match will take precedence. +::: + +:::{seealso} +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#abi-tag) for more information. +::: +""", + ), + "whl_platform_tags": attr.string_list( + doc = """\ +A list of `platform_tag` matchers so that we can select the best wheel based on the user +preference. +Will always include `"any"` even if it is not specified. + +The items in this list can contain a single `*` character that is equivalent to matching the +latest available version component in the platform_tag. Note, if the wheel platform tag does not +have a version component, e.g. `linux_x86_64` or `win_amd64`, then `*` will act as a regular +character. + +We will always select the highest available `platform_tag` version that is compatible with the +target platform. + +:::{note} +We select a single wheel and the last match will take precedence, if the platform_tag that we +match has a version component (e.g. `android_x_arch`, then the version `x` will be used in the +matching algorithm). + +If the matcher you provide has `*`, then we will match a wheel with the highest available target platform, i.e. if `musllinux_1_1_arch` and `musllinux_1_2_arch` are both present, then we will select `musllinux_1_2_arch`. +Otherwise we will select the highest available version that is equal or lower to the specifier, i.e. if `manylinux_2_12` and `manylinux_2_17` wheels are present and the matcher is `manylinux_2_15`, then we will match `manylinux_2_12` but not `manylinux_2_17`. +::: + +:::{note} +The following tag prefixes should be used instead of the legacy equivalents: +* `manylinux_2_5` instead of `manylinux1` +* `manylinux_2_12` instead of `manylinux2010` +* `manylinux_2_17` instead of `manylinux2014` + +When parsing the whl filenames `rules_python` will automatically transform wheel filenames to the +latest format. +::: + +:::{seealso} +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information. +::: +""", + ), } _SUPPORTED_PEP508_KEYS = [ diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 9c610f11d3..ebd447d95d 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -31,13 +31,14 @@ load("//python/private:repo_utils.bzl", "repo_utils") load(":index_sources.bzl", "index_sources") load(":parse_requirements_txt.bzl", "parse_requirements_txt") load(":pep508_requirement.bzl", "requirement") -load(":whl_target_platforms.bzl", "select_whls") +load(":select_whl.bzl", "select_whl") def parse_requirements( ctx, *, requirements_by_platform = {}, extra_pip_args = [], + platforms = {}, get_index_urls = None, evaluate_markers = None, extract_url_srcs = True, @@ -46,6 +47,7 @@ def parse_requirements( Args: ctx: A context that has .read function that would read contents from a label. + platforms: The target platform descriptions. requirements_by_platform (label_keyed_string_dict): a way to have different package versions (or different packages) for different os, arch combinations. @@ -88,7 +90,7 @@ def parse_requirements( requirements = {} for file, plats in requirements_by_platform.items(): if logger: - logger.debug(lambda: "Using {} for {}".format(file, plats)) + logger.trace(lambda: "Using {} for {}".format(file, plats)) contents = ctx.read(file) # Parse the requirements file directly in starlark to get the information @@ -161,7 +163,7 @@ def parse_requirements( # VCS package references. env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers) if logger: - logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( + logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( reqs_with_env_markers, env_marker_target_platforms, )) @@ -196,6 +198,7 @@ def parse_requirements( name = name, reqs = reqs, index_urls = index_urls, + platforms = platforms, env_marker_target_platforms = env_marker_target_platforms, extract_url_srcs = extract_url_srcs, logger = logger, @@ -203,7 +206,7 @@ def parse_requirements( ) ret.append(item) if not item.is_exposed and logger: - logger.debug(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( + logger.trace(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( name, sorted(requirement_target_platforms), sorted(requirements), @@ -219,38 +222,43 @@ def _package_srcs( name, reqs, index_urls, + platforms, logger, env_marker_target_platforms, extract_url_srcs): """A function to return sources for a particular package.""" srcs = {} for r in sorted(reqs.values(), key = lambda r: r.requirement_line): - whls, sdist = _add_dists( - requirement = r, - index_urls = index_urls.get(name), - logger = logger, - ) - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - target_platforms = sorted(target_platforms) + extra_pip_args = tuple(r.extra_pip_args) - all_dists = [] + whls - if sdist: - all_dists.append(sdist) + for target_platform in target_platforms: + if platforms and target_platform not in platforms: + fail("The target platform '{}' could not be found in {}".format( + target_platform, + platforms.keys(), + )) - if extract_url_srcs and all_dists: - req_line = r.srcs.requirement - else: - all_dists = [struct( - url = "", - filename = "", - sha256 = "", - yanked = False, - )] - req_line = r.srcs.requirement_line + dist = _add_dists( + requirement = r, + target_platform = platforms.get(target_platform), + index_urls = index_urls.get(name), + logger = logger, + ) + if logger: + logger.debug(lambda: "The whl dist is: {}".format(dist.filename if dist else dist)) + + if extract_url_srcs and dist: + req_line = r.srcs.requirement + else: + dist = struct( + url = "", + filename = "", + sha256 = "", + yanked = False, + ) + req_line = r.srcs.requirement_line - extra_pip_args = tuple(r.extra_pip_args) - for dist in all_dists: key = ( dist.filename, req_line, @@ -269,9 +277,9 @@ def _package_srcs( yanked = dist.yanked, ), ) - for p in target_platforms: - if p not in entry.target_platforms: - entry.target_platforms.append(p) + + if target_platform not in entry.target_platforms: + entry.target_platforms.append(target_platform) return srcs.values() @@ -325,7 +333,7 @@ def host_platform(ctx): repo_utils.get_platforms_cpu_name(ctx), ) -def _add_dists(*, requirement, index_urls, logger = None): +def _add_dists(*, requirement, index_urls, target_platform, logger = None): """Populate dists based on the information from the PyPI index. This function will modify the given requirements_by_platform data structure. @@ -333,6 +341,7 @@ def _add_dists(*, requirement, index_urls, logger = None): Args: requirement: The result of parse_requirements function. index_urls: The result of simpleapi_download. + target_platform: The target_platform information. logger: A logger for printing diagnostic info. """ @@ -342,7 +351,7 @@ def _add_dists(*, requirement, index_urls, logger = None): logger.debug(lambda: "Could not detect the filename from the URL, falling back to pip: {}".format( requirement.srcs.url, )) - return [], None + return None # Handle direct URLs in requirements dist = struct( @@ -353,12 +362,12 @@ def _add_dists(*, requirement, index_urls, logger = None): ) if dist.filename.endswith(".whl"): - return [dist], None + return dist else: - return [], dist + return dist if not index_urls: - return [], None + return None whls = [] sdist = None @@ -401,11 +410,16 @@ def _add_dists(*, requirement, index_urls, logger = None): for reason, dists in yanked.items() ])) - # Filter out the wheels that are incompatible with the target_platforms. - whls = select_whls( + if not target_platform: + # The pipstar platforms are undefined here, so we cannot do any matching + return sdist + + # Select a single wheel that can work on the target_platform + return select_whl( whls = whls, - want_platforms = requirement.target_platforms, + python_version = target_platform.env["python_full_version"], + implementation_name = target_platform.env["implementation_name"], + whl_abi_tags = target_platform.whl_abi_tags, + whl_platform_tags = target_platform.whl_platform_tags, logger = logger, - ) - - return whls, sdist + ) or sdist diff --git a/python/private/pypi/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl index 6ea3f120c3..6c3dd5da83 100644 --- a/python/private/pypi/whl_target_platforms.bzl +++ b/python/private/pypi/whl_target_platforms.bzl @@ -16,8 +16,6 @@ A starlark implementation of the wheel platform tag parsing to get the target platform. """ -load(":parse_whl_name.bzl", "parse_whl_name") - # The order of the dictionaries is to keep definitions with their aliases next to each # other _CPU_ALIASES = { @@ -46,136 +44,6 @@ _OS_PREFIXES = { "win": "windows", } # buildifier: disable=unsorted-dict-items -def select_whls(*, whls, want_platforms = [], logger = None): - """Select a subset of wheels suitable for target platforms from a list. - - Args: - whls(list[struct]): A list of candidates which have a `filename` - attribute containing the `whl` filename. - want_platforms(str): The platforms in "{abi}_{os}_{cpu}" or "{os}_{cpu}" format. - logger: A logger for printing diagnostic messages. - - Returns: - A filtered list of items from the `whls` arg where `filename` matches - the selected criteria. If no match is found, an empty list is returned. - """ - if not whls: - return [] - - want_abis = { - "abi3": None, - "none": None, - } - - _want_platforms = {} - version_limit = None - - for p in want_platforms: - if not p.startswith("cp3"): - fail("expected all platforms to start with ABI, but got: {}".format(p)) - - abi, _, os_cpu = p.partition("_") - abi, _, _ = abi.partition(".") - _want_platforms[os_cpu] = None - - # TODO @aignas 2025-04-20: add a test - _want_platforms["{}_{}".format(abi, os_cpu)] = None - - version_limit_candidate = int(abi[3:]) - if not version_limit: - version_limit = version_limit_candidate - if version_limit and version_limit != version_limit_candidate: - fail("Only a single python version is supported for now") - - # For some legacy implementations the wheels may target the `cp3xm` ABI - _want_platforms["{}m_{}".format(abi, os_cpu)] = None - want_abis[abi] = None - want_abis[abi + "m"] = None - - # Also add freethreaded wheels if we find them since we started supporting them - _want_platforms["{}t_{}".format(abi, os_cpu)] = None - want_abis[abi + "t"] = None - - want_platforms = sorted(_want_platforms) - - candidates = {} - for whl in whls: - parsed = parse_whl_name(whl.filename) - - if logger: - logger.trace(lambda: "Deciding whether to use '{}'".format(whl.filename)) - - supported_implementations = {} - whl_version_min = 0 - for tag in parsed.python_tag.split("."): - supported_implementations[tag[:2]] = None - - if tag.startswith("cp3") or tag.startswith("py3"): - version = int(tag[len("..3"):] or 0) - else: - # In this case it should be eithor "cp2" or "py2" and we will default - # to `whl_version_min` = 0 - continue - - if whl_version_min == 0 or version < whl_version_min: - whl_version_min = version - - if not ("cp" in supported_implementations or "py" in supported_implementations): - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support CPython, whl supported implementations are: {}".format(supported_implementations)) - continue - - if want_abis and parsed.abi_tag not in want_abis: - # Filter out incompatible ABIs - if logger: - logger.trace(lambda: "Discarding the whl because the whl abi did not match") - continue - - if whl_version_min > version_limit: - if logger: - logger.trace(lambda: "Discarding the whl because the whl supported python version is too high") - continue - - compatible = False - if parsed.platform_tag == "any": - compatible = True - else: - for p in whl_target_platforms(parsed.platform_tag, abi_tag = parsed.abi_tag.strip("m") if parsed.abi_tag.startswith("cp") else None): - if p.target_platform in want_platforms: - compatible = True - break - - if not compatible: - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support the desired platforms: {}".format(want_platforms)) - continue - - for implementation in supported_implementations: - candidates.setdefault( - ( - parsed.abi_tag, - parsed.platform_tag, - ), - {}, - ).setdefault( - ( - # prefer cp implementation - implementation == "cp", - # prefer higher versions - whl_version_min, - # prefer abi3 over none - parsed.abi_tag != "none", - # prefer cpx abi over abi3 - parsed.abi_tag != "abi3", - ), - [], - ).append(whl) - - return [ - candidates[key][sorted(v)[-1]][-1] - for key, v in candidates.items() - ] - def whl_target_platforms(platform_tag, abi_tag = ""): """Parse the wheel abi and platform tags and return (os, cpu) tuples. diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 04affd3adf..ea845216ac 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -65,13 +65,14 @@ def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_roo "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_platform_tags = whl_platform_tags, ) - for os, cpu in [ - ("linux", "x86_64"), - ("linux", "aarch64"), - ("osx", "aarch64"), - ("windows", "aarch64"), - ] + for (os, cpu), whl_platform_tags in { + ("linux", "x86_64"): ["linux_*_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64"): ["linux_*_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64"): ["macosx_*_arm64"], + ("windows", "aarch64"): ["win_arm64"], + }.items() ], ), is_root = is_root, @@ -105,21 +106,22 @@ def _build_config(env, enable_pipstar = 0, **kwargs): ) def _default( + *, arch_name = None, config_settings = None, os_name = None, platform = None, + whl_platform_tags = None, env = None, - whl_limit = None, - whl_platforms = None): + whl_abi_tags = None): return struct( arch_name = arch_name, os_name = os_name, platform = platform, + whl_platform_tags = whl_platform_tags or [], config_settings = config_settings, env = env or {}, - whl_platforms = whl_platforms, - whl_limit = whl_limit, + whl_abi_tags = whl_abi_tags or [], ) def _parse( @@ -417,18 +419,21 @@ def _test_torch_experimental_index_url(env): "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_platform_tags = whl_platform_tags, ) - for os, cpu in [ - ("linux", "aarch64"), - ("linux", "x86_64"), - ("osx", "aarch64"), - ("windows", "x86_64"), - ] + for (os, cpu), whl_platform_tags in { + ("linux", "x86_64"): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64"): ["linux_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64"): ["macosx_*_arm64"], + ("windows", "x86_64"): ["win_amd64"], + ("windows", "aarch64"): ["win_arm64"], # this should be ignored + }.items() ], parse = [ _parse( hub_name = "pypi", python_version = "3.12", + download_only = True, experimental_index_url = "https://torch.index", requirements_lock = "universal.txt", ), @@ -485,25 +490,25 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "torch": { "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ whl_config_setting( - filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", + target_platforms = ["cp312_linux_x86_64"], version = "3.12", ), ], "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ whl_config_setting( - filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + target_platforms = ["cp312_linux_aarch64"], version = "3.12", ), ], "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ whl_config_setting( - filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", + target_platforms = ["cp312_windows_x86_64"], version = "3.12", ), ], "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ whl_config_setting( - filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", + target_platforms = ["cp312_osx_aarch64"], version = "3.12", ), ], @@ -512,10 +517,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["linux_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -524,10 +526,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "osx_aarch64", - ], + "experimental_target_platforms": ["linux_aarch64"], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -536,10 +535,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["windows_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -548,10 +544,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "osx_aarch64", - ], + "experimental_target_platforms": ["osx_aarch64"], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -758,78 +751,79 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "pypi": { "direct_sdist_without_sha": { "pypi_315_any_name": [ - struct( - config_setting = None, - filename = "any-name.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "direct_without_sha": { "pypi_315_direct_without_sha_0_0_1_py3_none_any": [ - struct( - config_setting = None, - filename = "direct_without_sha-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "git_dep": { "pypi_315_git_dep": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "pip_fallback": { "pypi_315_pip_fallback": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "simple": { "pypi_315_simple_py3_none_any_deadb00f": [ - struct( - config_setting = None, - filename = "simple-0.0.1-py3-none-any.whl", - target_platforms = None, - version = "3.15", - ), - ], - "pypi_315_simple_sdist_deadbeef": [ - struct( - config_setting = None, - filename = "simple-0.0.1.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "some_other_pkg": { "pypi_315_some_py3_none_any_deadb33f": [ - struct( - config_setting = None, - filename = "some-other-pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "some_pkg": { "pypi_315_some_pkg_py3_none_any_deadbaaf": [ - struct( - config_setting = None, - filename = "some_pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], @@ -892,21 +886,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "sha256": "deadb00f", "urls": ["example2.org"], }, - "pypi_315_simple_sdist_deadbeef": { - "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], - "extra_pip_args": ["--extra-args-for-sdist-building"], - "filename": "simple-0.0.1.tar.gz", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1", - "sha256": "deadbeef", - "urls": ["example.org"], - }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ @@ -1153,7 +1132,9 @@ def _test_build_pipstar_platform(env): "@platforms//os:linux", "@platforms//cpu:x86_64", ], - env = {}, + env = {"implementation_name": "cpython"}, + whl_abi_tags = ["none", "abi3", "cp{major}{minor}"], + whl_platform_tags = ["any"], ), }) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 82fdd0a051..08bf3d3787 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -16,6 +16,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility def _mock_ctx(): testdata = { @@ -522,6 +523,26 @@ def _test_overlapping_shas_with_index_results(env): "requirements_linux": ["cp39_linux_x86_64"], "requirements_osx": ["cp39_osx_x86_64"], }, + platforms = { + "cp39_linux_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "linux", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["any"], + ), + "cp39_osx_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "osx", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["macosx_*_x86_64"], + ), + }, get_index_urls = lambda _, __: { "foo": struct( sdists = { @@ -563,20 +584,10 @@ def _test_overlapping_shas_with_index_results(env): filename = "foo-0.0.1-py3-none-any.whl", requirement_line = "foo==0.0.3", sha256 = "deadbaaf", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], + target_platforms = ["cp39_linux_x86_64"], url = "super2", yanked = False, ), - struct( - distribution = "foo", - extra_pip_args = [], - filename = "foo-0.0.1.tar.gz", - requirement_line = "foo==0.0.3", - sha256 = "5d15t", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], - url = "sdist", - yanked = False, - ), struct( distribution = "foo", extra_pip_args = [], diff --git a/tests/pypi/whl_target_platforms/BUILD.bazel b/tests/pypi/whl_target_platforms/BUILD.bazel index 6c35b08d32..fec25af033 100644 --- a/tests/pypi/whl_target_platforms/BUILD.bazel +++ b/tests/pypi/whl_target_platforms/BUILD.bazel @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":select_whl_tests.bzl", "select_whl_test_suite") load(":whl_target_platforms_tests.bzl", "whl_target_platforms_test_suite") -select_whl_test_suite(name = "select_whl_tests") - whl_target_platforms_test_suite(name = "whl_target_platforms_tests") diff --git a/tests/pypi/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl deleted file mode 100644 index 1674ac5ef2..0000000000 --- a/tests/pypi/whl_target_platforms/select_whl_tests.bzl +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"" - -load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "REPO_VERBOSITY_ENV_VAR", "repo_utils") # buildifier: disable=bzl-visibility -load("//python/private/pypi:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility - -WHL_LIST = [ - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp311-cp311-win32.whl", - "pkg-0.0.1-cp311-cp311-win_amd64.whl", - "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-win32.whl", - "pkg-0.0.1-cp37-cp37m-win_amd64.whl", - "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-cp39-win32.whl", - "pkg-0.0.1-cp39-cp39-win_amd64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py310-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", -] - -def _match(env, got, *want_filenames): - if not want_filenames: - env.expect.that_collection(got).has_size(len(want_filenames)) - return - - got_filenames = [g.filename for g in got] - env.expect.that_collection(got_filenames).contains_exactly(want_filenames) - - if got: - # Check that we pass the original structs - env.expect.that_str(got[0].other).equals("dummy") - -def _select_whls(whls, debug = False, **kwargs): - return select_whls( - whls = [ - struct( - filename = f, - other = "dummy", - ) - for f in whls - ], - logger = repo_utils.logger(struct( - os = struct( - environ = { - REPO_DEBUG_ENV_VAR: "1", - REPO_VERBOSITY_ENV_VAR: "TRACE" if debug else "INFO", - }, - ), - ), "unit-test"), - **kwargs - ) - -_tests = [] - -def _test_simplest(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ], - want_platforms = ["cp30_ignored"], - ) - _match( - env, - got, - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_simplest) - -def _test_select_by_supported_py_version(env): - for minor_version, match in { - 8: "pkg-0.0.1-py3-abi3-any.whl", - 11: "pkg-0.0.1-py311-abi3-any.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_select_by_supported_py_version) - -def _test_select_by_supported_cp_version(env): - for minor_version, match in { - 11: "pkg-0.0.1-cp311-abi3-any.whl", - 8: "pkg-0.0.1-py3-abi3-any.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - "pkg-0.0.1-cp311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_select_by_supported_cp_version) - -def _test_supported_cp_version_manylinux(env): - for minor_version, match in { - 8: "pkg-0.0.1-py3-none-manylinux_x86_64.whl", - 11: "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py311-none-manylinux_x86_64.whl", - "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", - ], - want_platforms = ["cp3{}_linux_x86_64".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_supported_cp_version_manylinux) - -def _test_ignore_unsupported(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-xx3-abi3-any.whl", - ], - want_platforms = ["cp30_ignored"], - ) - _match(env, got) - -_tests.append(_test_ignore_unsupported) - -def _test_match_abi_and_not_py_version(env): - # Check we match the ABI and not the py version - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp37_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_match_abi_and_not_py_version) - -def _test_select_filename_with_many_tags(env): - # Check we can select a filename with many platform tags - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_32"]) - _match( - env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_select_filename_with_many_tags) - -def _test_osx_prefer_arch_specific(env): - # Check that we prefer the specific wheel - got = _select_whls( - whls = WHL_LIST, - want_platforms = ["cp311_osx_x86_64", "cp311_osx_x86_32"], - ) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp311_osx_aarch64"]) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_osx_prefer_arch_specific) - -def _test_osx_fallback_to_universal2(env): - # Check that we can use the universal2 if the arm wheel is not available - got = _select_whls( - whls = [w for w in WHL_LIST if "arm64" not in w], - want_platforms = ["cp311_osx_aarch64"], - ) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_osx_fallback_to_universal2) - -def _test_prefer_manylinux_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_prefer_manylinux_wheels) - -def _test_freethreaded_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_freethreaded_wheels) - -def _test_micro_version_freethreaded(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313.3_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_micro_version_freethreaded) - -def select_whl_test_suite(name): - """Create the test suite. - - Args: - name: the name of the test suite - """ - test_suite(name = name, basic_tests = _tests) From 57a437287df7ff764e2e76ede02acfa40885f50b Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:49:40 +0900 Subject: [PATCH 6/6] feat(pypi): freethreaded support for the builder API DO NOT MERGE: stacked on #3058 This is a continuation of #3058 where we define freethreaded platforms. They need to be used only for particular python versions so I included an extra marker configuration attribute where we are using pipstar marker evaluation before using the platform. I think this in general will be a useful tool to configure only particular platforms for particular python versions Work towards #2548, since this shows how we can define custom platforms Work towards #2747 --- MODULE.bazel | 36 +++++++-- python/private/pypi/extension.bzl | 77 +++++++++++++------ python/private/pypi/pip_repository.bzl | 7 +- .../pypi/requirements_files_by_platform.bzl | 8 +- .../resolve_target_platforms.py | 4 +- tests/pypi/extension/extension_tests.bzl | 33 +++++--- 6 files changed, 119 insertions(+), 46 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 97de1ad6eb..1ce95e9f23 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -70,11 +70,15 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") config_settings = [ "@platforms//cpu:{}".format(cpu), "@platforms//os:linux", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), ], env = {"platform_version": "0"}, + marker = "python_version >= '3.13'" if freethreaded else "", os_name = "linux", - platform = "linux_{}".format(cpu), - whl_abi_tags = [ + platform = "linux_{}{}".format(cpu, freethreaded), + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ "abi3", "cp{major}{minor}", ], @@ -87,6 +91,10 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") "x86_64", "aarch64", ] + for freethreaded in [ + "", + "_freethreaded", + ] ] [ @@ -95,13 +103,17 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") config_settings = [ "@platforms//cpu:{}".format(cpu), "@platforms//os:osx", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), ], # We choose the oldest non-EOL version at the time when we release `rules_python`. # See https://endoflife.date/macos env = {"platform_version": "14.0"}, + marker = "python_version >= '3.13'" if freethreaded else "", os_name = "osx", - platform = "osx_{}".format(cpu), - whl_abi_tags = [ + platform = "osx_{}{}".format(cpu, freethreaded), + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ "abi3", "cp{major}{minor}", ], @@ -120,6 +132,10 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") "x86_64", ], }.items() + for freethreaded in [ + "", + "_freethreaded", + ] ] [ @@ -128,11 +144,15 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") config_settings = [ "@platforms//cpu:{}".format(cpu), "@platforms//os:windows", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), ], env = {"platform_version": "0"}, + marker = "python_version >= '3.13'" if freethreaded else "", os_name = "windows", - platform = "windows_{}".format(cpu), - whl_abi_tags = [ + platform = "windows_{}{}".format(cpu, freethreaded), + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ "abi3", "cp{major}{minor}", ], @@ -141,6 +161,10 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") for cpu, whl_platform_tags in { "x86_64": ["win_amd64"], }.items() + for freethreaded in [ + "", + "_freethreaded", + ] ] pip.parse( diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index c16c3c6de0..607ea4c767 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -30,6 +30,7 @@ load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") +load(":pep508_evaluate.bzl", "evaluate") load(":pip_repository_attrs.bzl", "ATTRS") load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") @@ -80,21 +81,27 @@ def _platforms(*, python_version, minor_mapping, config): for platform, values in config.platforms.items(): # TODO @aignas 2025-07-07: this is probably doing the parsing of the version too # many times. - key = "{}{}{}.{}_{}".format( + abi = "{}{}{}.{}".format( python_tag(values.env["implementation_name"]), python_version.release[0], python_version.release[1], python_version.release[2], - platform, ) + key = "{}_{}".format(abi, platform) + + env_ = env( + env = values.env, + os = values.os_name, + arch = values.arch_name, + python_version = python_version.string, + ) + + if values.marker and not evaluate(values.marker, env = env_): + continue platforms[key] = struct( - env = env( - env = values.env, - os = values.os_name, - arch = values.arch_name, - python_version = python_version.string, - ), + env = env_, + triple = "{}_{}_{}".format(abi, values.os_name, values.arch_name), whl_abi_tags = [ v.format( major = python_version.release[0], @@ -203,17 +210,19 @@ def _create_whl_repos( whl_group_mapping = {} requirement_cycles = {} + platforms = _platforms( + python_version = pip_attr.python_version, + minor_mapping = minor_mapping, + config = config, + ) + if evaluate_markers: # This is most likely unit tests pass elif config.enable_pipstar: evaluate_markers = lambda _, requirements: evaluate_markers_star( requirements = requirements, - platforms = _platforms( - python_version = pip_attr.python_version, - minor_mapping = minor_mapping, - config = config, - ), + platforms = platforms, ) else: # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either @@ -232,7 +241,13 @@ def _create_whl_repos( # spin up a Python interpreter. evaluate_markers = lambda module_ctx, requirements: evaluate_markers_py( module_ctx, - requirements = requirements, + requirements = { + k: { + p: platforms[p].triple + for p in plats + } + for k, plats in requirements.items() + }, python_interpreter = pip_attr.python_interpreter, python_interpreter_target = python_interpreter_target, srcs = pip_attr._evaluate_markers_srcs, @@ -248,18 +263,14 @@ def _create_whl_repos( requirements_osx = pip_attr.requirements_darwin, requirements_windows = pip_attr.requirements_windows, extra_pip_args = pip_attr.extra_pip_args, - platforms = sorted(config.platforms), # here we only need keys + platforms = sorted(platforms), # here we only need keys python_version = full_version( version = pip_attr.python_version, minor_mapping = minor_mapping, ), logger = logger, ), - platforms = _platforms( - python_version = pip_attr.python_version, - minor_mapping = minor_mapping, - config = config, - ), + platforms = platforms, extra_pip_args = pip_attr.extra_pip_args, get_index_urls = get_index_urls, evaluate_markers = evaluate_markers, @@ -346,6 +357,16 @@ def _create_whl_repos( )) whl_libraries[repo_name] = repo.args + if not config.enable_pipstar and "experimental_target_platforms" in repo.args: + whl_libraries[repo_name] |= { + "experimental_target_platforms": sorted({ + # TODO @aignas 2025-07-07: this should be solved in a better way + platforms[candidate].triple.partition("_")[-1]: None + for p in repo.args["experimental_target_platforms"] + for candidate in platforms + if candidate.endswith(p) + }), + } whl_map.setdefault(whl.name, {})[repo.config_setting] = repo_name return struct( @@ -434,6 +455,7 @@ def _configure( arch_name, config_settings, env = {}, + marker, whl_abi_tags, whl_platform_tags, override = False): @@ -441,7 +463,7 @@ def _configure( config.setdefault("platforms", {}) if platform and ( - os_name or arch_name or config_settings or whl_abi_tags or whl_platform_tags or env + os_name or arch_name or config_settings or whl_abi_tags or whl_platform_tags or env or marker ): if not override and config["platforms"].get(platform): return @@ -455,6 +477,7 @@ def _configure( "arch_name": arch_name, "config_settings": config_settings, "env": env, + "marker": marker, "name": platform.replace("-", "_").lower(), "os_name": os_name, "whl_abi_tags": whl_abi_tags, @@ -470,7 +493,7 @@ def _configure( else: config["platforms"].pop(platform) -def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, whl_abi_tags = [], whl_platform_tags = []): +def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, marker = "", whl_abi_tags = [], whl_platform_tags = []): # NOTE @aignas 2025-07-08: the least preferred is the first item in the list if "any" not in whl_platform_tags: # the lowest priority one needs to be the first one @@ -490,6 +513,7 @@ def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, whl_abi_t # defaults for env "implementation_name": "cpython", } | env, + marker = marker, whl_abi_tags = whl_abi_tags, whl_platform_tags = whl_platform_tags, ) @@ -524,6 +548,7 @@ def build_config( config_settings = tag.config_settings, env = tag.env, os_name = tag.os_name, + marker = tag.marker, platform = platform, override = mod.is_root, whl_abi_tags = tag.whl_abi_tags, @@ -533,8 +558,6 @@ def build_config( # attribute. # * for index/downloader config. This includes all of those attributes for # overrides, etc. Index overrides per platform could be also used here. - # * for whl selection - selecting preferences of which `platform_tag`s we should use - # for what. We could also model the `cp313t` freethreaded as separate platforms. ) return struct( @@ -918,6 +941,12 @@ Supported keys: ::::{note} This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled. :::: +""", + ), + "marker": attr.string( + doc = """\ +A marker which will be evaluated to disable the target platform for certain python versions. This +is especially useful when defining freethreaded platform variants. """, ), # The values for PEP508 env marker evaluation during the lock file parsing diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index e63bd6c3d1..3df56f24ff 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -94,7 +94,12 @@ def _pip_repository_impl(rctx): extra_pip_args = rctx.attr.extra_pip_args, evaluate_markers = lambda rctx, requirements: evaluate_markers_py( rctx, - requirements = requirements, + requirements = { + # NOTE @aignas 2025-07-07: because we don't distinguish between + # freethreaded and non-freethreaded, it is a 1:1 mapping. + req: {p: p for p in plats} + for req, plats in requirements.items() + }, python_interpreter = rctx.attr.python_interpreter, python_interpreter_target = rctx.attr.python_interpreter_target, srcs = rctx.attr._evaluate_markers_srcs, diff --git a/python/private/pypi/requirements_files_by_platform.bzl b/python/private/pypi/requirements_files_by_platform.bzl index d8d3651461..356bd4416e 100644 --- a/python/private/pypi/requirements_files_by_platform.bzl +++ b/python/private/pypi/requirements_files_by_platform.bzl @@ -37,7 +37,9 @@ def _default_platforms(*, filter, platforms): if not prefix: return platforms - match = [p for p in platforms if p.startswith(prefix)] + match = [p for p in platforms if p.startswith(prefix) or ( + p.startswith("cp") and p.partition("_")[-1].startswith(prefix) + )] else: match = [p for p in platforms if filter in p] @@ -140,7 +142,7 @@ def requirements_files_by_platform( if logger: logger.debug(lambda: "Platforms from pip args: {}".format(platforms_from_args)) - default_platforms = [_platform(p, python_version) for p in platforms] + default_platforms = platforms if platforms_from_args: lock_files = [ @@ -252,6 +254,6 @@ def requirements_files_by_platform( ret = {} for plat, file in requirements.items(): - ret.setdefault(file, []).append(plat) + ret.setdefault(file, []).append(_platform(plat, python_version = python_version)) return ret diff --git a/python/private/pypi/requirements_parser/resolve_target_platforms.py b/python/private/pypi/requirements_parser/resolve_target_platforms.py index c899a943cc..accacf5bfa 100755 --- a/python/private/pypi/requirements_parser/resolve_target_platforms.py +++ b/python/private/pypi/requirements_parser/resolve_target_platforms.py @@ -50,8 +50,8 @@ def main(): hashes = prefix + hashes req = Requirement(entry) - for p in target_platforms: - (platform,) = Platform.from_string(p) + for p, triple in target_platforms.items(): + (platform,) = Platform.from_string(triple) if not req.marker or req.marker.evaluate(platform.env_markers("")): response.setdefault(requirement_line, []).append(p) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ea845216ac..999d4d154c 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -58,20 +58,22 @@ def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_roo whl_mods = whl_mods, default = default or [ _default( - platform = "{}_{}".format(os, cpu), + platform = "{}_{}{}".format(os, cpu, freethreaded), os_name = os, arch_name = cpu, config_settings = [ "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else ["abi3", "cp{major}{minor}"], whl_platform_tags = whl_platform_tags, ) - for (os, cpu), whl_platform_tags in { - ("linux", "x86_64"): ["linux_*_x86_64", "manylinux_*_x86_64"], - ("linux", "aarch64"): ["linux_*_aarch64", "manylinux_*_aarch64"], - ("osx", "aarch64"): ["macosx_*_arm64"], - ("windows", "aarch64"): ["win_arm64"], + for (os, cpu, freethreaded), whl_platform_tags in { + ("linux", "x86_64", ""): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "x86_64", "_freethreaded"): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64", ""): ["linux_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64", ""): ["macosx_*_arm64"], + ("windows", "aarch64", ""): ["win_arm64"], }.items() ], ), @@ -113,6 +115,7 @@ def _default( platform = None, whl_platform_tags = None, env = None, + marker = None, whl_abi_tags = None): return struct( arch_name = arch_name, @@ -121,6 +124,7 @@ def _default( whl_platform_tags = whl_platform_tags or [], config_settings = config_settings, env = env or {}, + marker = marker or "", whl_abi_tags = whl_abi_tags or [], ) @@ -349,10 +353,11 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ version = "3.15", ), ], - "pypi_315_torch_linux_x86_64": [ + "pypi_315_torch_linux_x86_64_linux_x86_64_freethreaded": [ whl_config_setting( target_platforms = [ "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", ], version = "3.15", ), @@ -365,7 +370,7 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1 --hash=sha256:deadbeef", }, - "pypi_315_torch_linux_x86_64": { + "pypi_315_torch_linux_x86_64_linux_x86_64_freethreaded": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -755,6 +760,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef target_platforms = ( "cp315_linux_aarch64", "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", "cp315_osx_aarch64", "cp315_windows_aarch64", ), @@ -768,6 +774,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef target_platforms = ( "cp315_linux_aarch64", "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", "cp315_osx_aarch64", "cp315_windows_aarch64", ), @@ -795,6 +802,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef target_platforms = ( "cp315_linux_aarch64", "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", "cp315_osx_aarch64", "cp315_windows_aarch64", ), @@ -808,6 +816,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef target_platforms = ( "cp315_linux_aarch64", "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", "cp315_osx_aarch64", "cp315_windows_aarch64", ), @@ -821,6 +830,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef target_platforms = ( "cp315_linux_aarch64", "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", "cp315_osx_aarch64", "cp315_windows_aarch64", ), @@ -974,12 +984,13 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.hub_whl_map().contains_exactly({ "pypi": { "optimum": { - "pypi_315_optimum_linux_aarch64_linux_x86_64": [ + "pypi_315_optimum_linux_aarch64_linux_x86_64_linux_x86_64_freethreaded": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_linux_aarch64", "cp315_linux_x86_64", + "cp315_linux_x86_64_freethreaded", ], config_setting = None, filename = None, @@ -1000,7 +1011,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_linux_aarch64_linux_x86_64": { + "pypi_315_optimum_linux_aarch64_linux_x86_64_linux_x86_64_freethreaded": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", @@ -1026,6 +1037,7 @@ def _test_pipstar_platforms(env): platform = "my{}_{}".format(os, cpu), os_name = os, arch_name = cpu, + marker = "python_version ~= \"3.13\"", config_settings = [ "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), @@ -1133,6 +1145,7 @@ def _test_build_pipstar_platform(env): "@platforms//cpu:x86_64", ], env = {"implementation_name": "cpython"}, + marker = "", whl_abi_tags = ["none", "abi3", "cp{major}{minor}"], whl_platform_tags = ["any"], ),