From d04407b98b511030f25ad94f38fb62267bced86f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 6 Jan 2021 10:20:56 -0500 Subject: [PATCH 1/6] feat: support pypa build fix: update `cibuildwheel.linux.troubleshoot` to know about build Workaround issue with PyPy venv module by installing build[virtualenv] fix: test_dependency_constraints_file when using build fix: test/test_pep518.py fix: use `strtobool` to parse `CIBW_PYPA_BUILD` fix: update `cibuildwheel.linux.troubleshoot` to know about `python -m pip wheel` test: use `build_mode` in `test/test_dependency_versions.py` tests --- cibuildwheel/__main__.py | 3 + cibuildwheel/linux.py | 49 +++++++++++----- cibuildwheel/macos.py | 99 +++++++++++++++++++++++--------- cibuildwheel/util.py | 18 ++++++ cibuildwheel/windows.py | 85 +++++++++++++++++++++------ test/conftest.py | 11 +++- test/test_0_basic.py | 4 +- test/test_before_build.py | 7 ++- test/test_dependency_versions.py | 9 ++- test/test_pep518.py | 15 +++-- test/test_troubleshooting.py | 4 +- 11 files changed, 230 insertions(+), 74 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 1d3ce40c4..9208f5d45 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -27,6 +27,7 @@ Unbuffered, detect_ci_provider, resources_dir, + strtobool, ) MANYLINUX_ARCHS = ( @@ -181,6 +182,7 @@ def main() -> None: build_config = options("build", env_plat=False, sep=" ") or "*" skip_config = options("skip", env_plat=False, sep=" ") test_skip = options("test-skip", env_plat=False, sep=" ") + pypa_build = strtobool(os.environ.get("CIBW_PYPA_BUILD", "0")) archs_config_str = args.archs or options("archs", sep=" ") @@ -308,6 +310,7 @@ def main() -> None: environment=environment, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images or None, + pypa_build=pypa_build, ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index faeff20b4..9b6a60799 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -145,7 +145,7 @@ def build(options: BuildOptions) -> None: env, executor=docker.environment_executor ) - # check config python and pip are still on PATH + # check config python is still on PATH which_python = docker.call( ["which", "python"], env=env, capture_output=True ).strip() @@ -180,18 +180,36 @@ def build(options: BuildOptions) -> None: docker.call(["rm", "-rf", built_wheel_dir]) docker.call(["mkdir", "-p", built_wheel_dir]) - docker.call( - [ - "pip", - "wheel", - container_package_dir, - "--wheel-dir", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.pypa_build: + config_setting = " ".join(verbosity_flags) + docker.call( + [ + "python", + "-m", + "build", + container_package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=env, + ) + else: + docker.call( + [ + "python", + "-m", + "pip", + "wheel", + container_package_dir, + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, + ], + env=env, + ) built_wheel = docker.glob(built_wheel_dir, "*.whl")[0] @@ -291,7 +309,10 @@ def build(options: BuildOptions) -> None: def troubleshoot(package_dir: Path, error: Exception) -> None: - if isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ["pip", "wheel"]: + if isinstance(error, subprocess.CalledProcessError) and ( + error.cmd[0:4] == ["python", "-m", "pip", "wheel"] + or error.cmd[0:3] == ["python", "-m", "build"] + ): # the 'pip wheel' step failed. print("Checking for common errors...") so_files = list(package_dir.glob("**/*.so")) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index cbaeae61d..4a055f75c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -19,6 +19,7 @@ NonPlatformWheelError, download, get_build_verbosity_extra_flags, + get_pip_version, install_certifi_script, prepare_command, read_python_configs, @@ -178,7 +179,9 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, + pypa_build: bool, ) -> Dict[str, str]: + implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") @@ -308,18 +311,31 @@ def setup_python( env.setdefault("SDKROOT", arm64_compatible_sdks[0]) log.step("Installing build tools...") - call( - [ - "pip", - "install", - "--upgrade", - "setuptools", - "wheel", - "delocate", - *dependency_constraint_flags, - ], - env=env, - ) + if pypa_build: + call( + [ + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv]", + *dependency_constraint_flags, + ], + env=env, + ) + else: + call( + [ + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + "delocate", + *dependency_constraint_flags, + ], + env=env, + ) return env @@ -356,7 +372,12 @@ def build(options: BuildOptions) -> None: options.dependency_constraints.get_for_python_version(config.version), ] - env = setup_python(config, dependency_constraint_flags, options.environment) + env = setup_python( + config, + dependency_constraint_flags, + options.environment, + options.pypa_build, + ) if options.before_build: log.step("Running before_build...") @@ -370,20 +391,44 @@ def build(options: BuildOptions) -> None: shutil.rmtree(built_wheel_dir) built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "pip", - "wheel", - options.package_dir.resolve(), - "--wheel-dir", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.pypa_build: + config_setting = " ".join(verbosity_flags) + build_env = dict(env) + if options.dependency_constraints: + build_env["PIP_CONSTRAINT"] = str( + options.dependency_constraints.get_for_python_version(config.version) + ) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + [ + "python", + "-m", + "build", + options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=build_env, + ) + else: + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, + ], + env=env, + ) built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 5d5c47683..f751956ce 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -4,6 +4,8 @@ import os import re import ssl +import subprocess +import sys import textwrap import time import urllib.request @@ -218,6 +220,7 @@ class BuildOptions(NamedTuple): test_requires: List[str] test_extras: str build_verbosity: int + pypa_build: bool class NonPlatformWheelError(Exception): @@ -300,3 +303,18 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]: s = time.time() - start_time m = s / 60 print(msg.format(n=n, s=s, m=m), *sorted(f" {f.name}" for f in new_contents), sep="\n") + + +def get_pip_version(env: Dict[str, str]) -> str: + # we use shell=True here for windows, even though we don't need a shell due to a bug + # https://bugs.python.org/issue8557 + shell = sys.platform.startswith("win") + versions_output_text = subprocess.check_output( + ["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env + ) + (pip_version,) = [ + version[5:] + for version in versions_output_text.strip().splitlines() + if version.startswith("pip==") + ] + return pip_version diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 9ce8e309f..3cc4c0cef 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -17,6 +17,7 @@ NonPlatformWheelError, download, get_build_verbosity_extra_flags, + get_pip_version, prepare_command, read_python_configs, ) @@ -114,7 +115,9 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, + pypa_build: bool, ) -> Dict[str, str]: + nuget = Path("C:\\cibw\\nuget.exe") if not nuget.exists(): log.step("Downloading nuget...") @@ -214,10 +217,24 @@ def setup_python( sys.exit(1) call(["pip", "--version"], env=env) - call( - ["pip", "install", "--upgrade", "setuptools", "wheel", *dependency_constraint_flags], - env=env, - ) + + if pypa_build: + call( + ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags], + env=env, + ) + else: + call( + [ + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + *dependency_constraint_flags, + ], + env=env, + ) return env @@ -251,7 +268,12 @@ def build(options: BuildOptions) -> None: ] # install Python - env = setup_python(config, dependency_constraint_flags, options.environment) + env = setup_python( + config, + dependency_constraint_flags, + options.environment, + options.pypa_build, + ) # run the before_build command if options.before_build: @@ -265,20 +287,45 @@ def build(options: BuildOptions) -> None: if built_wheel_dir.exists(): shutil.rmtree(built_wheel_dir) built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "pip", - "wheel", - options.package_dir.resolve(), - "-w", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.pypa_build: + config_setting = " ".join(verbosity_flags) + build_env = dict(env) + if options.dependency_constraints: + build_env["PIP_CONSTRAINT"] = str( + options.dependency_constraints.get_for_python_version(config.version) + ) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + [ + "python", + "-m", + "build", + options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=build_env, + ) + else: + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *get_build_verbosity_extra_flags(options.build_verbosity), + ], + env=env, + ) built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/test/conftest.py b/test/conftest.py index 410c07076..ce433b8a0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,7 +1,9 @@ +from typing import Dict + import pytest -def pytest_addoption(parser): +def pytest_addoption(parser) -> None: parser.addoption( "--run-emulation", action="store_true", default=False, help="run emulation tests" ) @@ -11,7 +13,7 @@ def pytest_configure(config): config.addinivalue_line("markers", "emulation: mark test requiring qemu binfmt_misc to run") -def pytest_collection_modifyitems(config, items): +def pytest_collection_modifyitems(config, items) -> None: if config.getoption("--run-emulation"): # --run-emulation given in cli: do not skip emulation tests return @@ -19,3 +21,8 @@ def pytest_collection_modifyitems(config, items): for item in items: if "emulation" in item.keywords: item.add_marker(skip_emulation) + + +@pytest.fixture(params=[{"CIBW_PYPA_BUILD": "0"}, {"CIBW_PYPA_BUILD": "1"}], ids=["pip", "pypa"]) +def build_mode(request) -> Dict[str, str]: + return request.param diff --git a/test/test_0_basic.py b/test/test_0_basic.py index a84532261..d012fe06f 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -18,12 +18,12 @@ ) -def test(tmp_path): +def test(tmp_path, build_mode): project_dir = tmp_path / "project" basic_project.generate(project_dir) # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir) + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_mode) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") diff --git a/test/test_before_build.py b/test/test_before_build.py index f84bd8007..c451abb62 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -25,8 +25,13 @@ stored_executable = f.read() print('stored_executable', stored_executable) print('sys.executable', sys.executable) + # windows/mac are case insensitive - assert os.path.realpath(stored_executable).lower() == os.path.realpath(sys.executable).lower() + stored_path = os.path.realpath(stored_executable).lower() + current_path = os.path.realpath(sys.executable).lower() + + # TODO: This is not valid in an virtual environment + # assert stored_path == current_path, '{0} != {1}'.format(stored_path, current_path) """ ) ) diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index dc1f0b50c..76a2dd866 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -48,7 +48,7 @@ def get_versions_from_constraint_file(constraint_file): @pytest.mark.parametrize("python_version", ["3.6", "3.8", "3.9"]) -def test_pinned_versions(tmp_path, python_version): +def test_pinned_versions(tmp_path, python_version, build_mode): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -85,6 +85,7 @@ def test_pinned_versions(tmp_path, python_version): add_env={ "CIBW_BUILD": build_pattern, "CIBW_ENVIRONMENT": cibw_environment_option, + **build_mode, }, ) @@ -107,7 +108,7 @@ def test_pinned_versions(tmp_path, python_version): assert set(actual_wheels) == set(expected_wheels) -def test_dependency_constraints_file(tmp_path): +def test_dependency_constraints_file(tmp_path, build_mode): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -118,7 +119,7 @@ def test_dependency_constraints_file(tmp_path): "pip": "20.0.2", "setuptools": "53.0.0", "wheel": "0.34.2", - "virtualenv": "20.0.10", + "virtualenv": "20.0.35", } constraints_file = tmp_path / "constraints.txt" @@ -129,6 +130,7 @@ def test_dependency_constraints_file(tmp_path): setuptools=={setuptools} wheel=={wheel} virtualenv=={virtualenv} + importlib-metadata<3,>=0.12; python_version < "3.8" """.format( **tool_versions ) @@ -149,6 +151,7 @@ def test_dependency_constraints_file(tmp_path): add_env={ "CIBW_ENVIRONMENT": cibw_environment_option, "CIBW_DEPENDENCY_VERSIONS": str(constraints_file), + **build_mode, }, ) diff --git a/test/test_pep518.py b/test/test_pep518.py index 921024f55..33be7d273 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -1,4 +1,3 @@ -import os import textwrap from . import test_projects, utils @@ -33,13 +32,13 @@ """ -def test_pep518(tmp_path): +def test_pep518(tmp_path, build_mode): project_dir = tmp_path / "project" basic_project.generate(project_dir) # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir) + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_mode) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") @@ -50,4 +49,12 @@ def test_pep518(tmp_path): assert not (project_dir / "42").exists() assert not (project_dir / "4.1.2").exists() - assert len(os.listdir(project_dir)) == len(basic_project.files) + # pypa/build creates a "build" folder & a "*.egg-info" folder for the wheel being built, + # this should be harmless so remove them + contents = [ + item + for item in project_dir.iterdir() + if item.name != "build" and not item.name.endswith(".egg-info") + ] + + assert len(contents) == len(basic_project.files) diff --git a/test/test_troubleshooting.py b/test/test_troubleshooting.py index e1d2cf8d5..e1ccf48d6 100644 --- a/test/test_troubleshooting.py +++ b/test/test_troubleshooting.py @@ -16,7 +16,7 @@ """ -def test_failed_project_with_so_files(tmp_path, capfd): +def test_failed_project_with_so_files(tmp_path, capfd, build_mode): if utils.platform != "linux": pytest.skip("this test is only relevant to the linux build") @@ -24,7 +24,7 @@ def test_failed_project_with_so_files(tmp_path, capfd): so_file_project.generate(project_dir) with pytest.raises(subprocess.CalledProcessError): - utils.cibuildwheel_run(project_dir) + utils.cibuildwheel_run(project_dir, add_env=build_mode) captured = capfd.readouterr() print("out", captured.out) From f7d31fc0eea04d14abf3a2af3ed43d902544f5ba Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 21 Jun 2021 15:23:18 -0400 Subject: [PATCH 2/6] refactor: use build-backend instead --- cibuildwheel/__main__.py | 12 +++++-- cibuildwheel/linux.py | 28 ++++++++------- cibuildwheel/macos.py | 50 ++++++++++++++------------ cibuildwheel/resources/defaults.toml | 1 + cibuildwheel/util.py | 6 ++-- cibuildwheel/windows.py | 52 +++++++++++++++------------- docs/options.md | 30 ++++++++++++++++ test/conftest.py | 6 ++-- 8 files changed, 118 insertions(+), 67 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 9208f5d45..4e5b9a00b 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -27,7 +27,6 @@ Unbuffered, detect_ci_provider, resources_dir, - strtobool, ) MANYLINUX_ARCHS = ( @@ -41,6 +40,8 @@ "pypy_i686", ) +BUILD_FRONTENDS = {"pip", "build"} + def main() -> None: platform: PlatformName @@ -182,10 +183,10 @@ def main() -> None: build_config = options("build", env_plat=False, sep=" ") or "*" skip_config = options("skip", env_plat=False, sep=" ") test_skip = options("test-skip", env_plat=False, sep=" ") - pypa_build = strtobool(os.environ.get("CIBW_PYPA_BUILD", "0")) archs_config_str = args.archs or options("archs", sep=" ") + build_frontend = options("build-frontend", env_plat=False) environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "}) before_all = options("before-all", sep=" && ") before_build = options("before-build", sep=" && ") @@ -202,6 +203,11 @@ def main() -> None: os.environ.get("CIBW_PRERELEASE_PYTHONS", "0") ) + if build_frontend not in BUILD_FRONTENDS: + msg = f"cibuildwheel: Unrecognised build front end '{build_frontend}', only {BUILD_FRONTENDS} supported" + print(msg, file=sys.stderr) + sys.exit(2) + package_files = {"setup.py", "setup.cfg", "pyproject.toml"} if not any(package_dir.joinpath(name).exists() for name in package_files): @@ -310,7 +316,7 @@ def main() -> None: environment=environment, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images or None, - pypa_build=pypa_build, + build_frontend=build_frontend, ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 9b6a60799..d2bf56ab9 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -182,34 +182,38 @@ def build(options: BuildOptions) -> None: verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) - if options.pypa_build: - config_setting = " ".join(verbosity_flags) + if options.build_frontend == "pip": docker.call( [ "python", "-m", - "build", + "pip", + "wheel", container_package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, ], env=env, ) - else: + elif options.build_frontend == "build": + config_setting = " ".join(verbosity_flags) docker.call( [ "python", "-m", - "pip", - "wheel", + "build", container_package_dir, - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *verbosity_flags, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", ], env=env, ) + else: + raise RuntimeError( + f"build_frontend {options.build_frontend!r} not understood" + ) built_wheel = docker.glob(built_wheel_dir, "*.whl")[0] diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 4a055f75c..5c94b34d5 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -179,7 +179,7 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, - pypa_build: bool, + build_frontend: str, ) -> Dict[str, str]: implementation_id = python_configuration.identifier.split("-")[0] @@ -311,31 +311,33 @@ def setup_python( env.setdefault("SDKROOT", arm64_compatible_sdks[0]) log.step("Installing build tools...") - if pypa_build: + if build_frontend == "pip": call( [ "pip", "install", "--upgrade", + "setuptools", + "wheel", "delocate", - "build[virtualenv]", *dependency_constraint_flags, ], env=env, ) - else: + elif build_frontend == "build": call( [ "pip", "install", "--upgrade", - "setuptools", - "wheel", "delocate", + "build[virtualenv]", *dependency_constraint_flags, ], env=env, ) + else: + raise RuntimeError(f"build_frontend {build_frontend!r} not understood") return env @@ -376,7 +378,7 @@ def build(options: BuildOptions) -> None: config, dependency_constraint_flags, options.environment, - options.pypa_build, + options.build_frontend, ) if options.before_build: @@ -393,7 +395,23 @@ def build(options: BuildOptions) -> None: verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) - if options.pypa_build: + if options.build_frontend == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, + ], + env=env, + ) + elif options.build_frontend == "build": config_setting = " ".join(verbosity_flags) build_env = dict(env) if options.dependency_constraints: @@ -414,21 +432,7 @@ def build(options: BuildOptions) -> None: env=build_env, ) else: - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "python", - "-m", - "pip", - "wheel", - options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *verbosity_flags, - ], - env=env, - ) + raise RuntimeError(f"build_frontend {options.build_frontend!r} not understood") built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index a9ecff021..5bf179c89 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -4,6 +4,7 @@ skip = "" test-skip = "" archs = ["auto"] +build-frontend = "pip" dependency-versions = "pinned" environment = {} build-verbosity = "" diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index f751956ce..e7e5a675a 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -220,7 +220,7 @@ class BuildOptions(NamedTuple): test_requires: List[str] test_extras: str build_verbosity: int - pypa_build: bool + build_frontend: str class NonPlatformWheelError(Exception): @@ -312,9 +312,9 @@ def get_pip_version(env: Dict[str, str]) -> str: versions_output_text = subprocess.check_output( ["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env ) - (pip_version,) = [ + (pip_version,) = ( version[5:] for version in versions_output_text.strip().splitlines() if version.startswith("pip==") - ] + ) return pip_version diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 3cc4c0cef..188efd805 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -115,7 +115,7 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, - pypa_build: bool, + build_frontend: str, ) -> Dict[str, str]: nuget = Path("C:\\cibw\\nuget.exe") @@ -218,12 +218,7 @@ def setup_python( call(["pip", "--version"], env=env) - if pypa_build: - call( - ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags], - env=env, - ) - else: + if build_frontend == "pip": call( [ "pip", @@ -235,6 +230,13 @@ def setup_python( ], env=env, ) + elif build_frontend == "build": + call( + ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags], + env=env, + ) + else: + raise RuntimeError(f"build_frontend {build_frontend!r} not understood") return env @@ -272,7 +274,7 @@ def build(options: BuildOptions) -> None: config, dependency_constraint_flags, options.environment, - options.pypa_build, + options.build_frontend, ) # run the before_build command @@ -290,7 +292,23 @@ def build(options: BuildOptions) -> None: verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) - if options.pypa_build: + if options.build_frontend == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *get_build_verbosity_extra_flags(options.build_verbosity), + ], + env=env, + ) + elif options.build_frontend == "build": config_setting = " ".join(verbosity_flags) build_env = dict(env) if options.dependency_constraints: @@ -311,21 +329,7 @@ def build(options: BuildOptions) -> None: env=build_env, ) else: - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "python", - "-m", - "pip", - "wheel", - options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + raise RuntimeError(f"build_frontend {options.build_frontend!r} not understood") built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/docs/options.md b/docs/options.md index 832a8fc4d..60c427b61 100644 --- a/docs/options.md +++ b/docs/options.md @@ -435,6 +435,36 @@ This option can also be set using the [command-line option](#command-line) `--pr ## Build customization +### `CIBW_BUILD_FRONTEND` {: #build-frontend} +> Set the tool to use to build, either "pip" (default for now) or "build" + +A selector for the build backend to use. Can either be "pip", which will run +`python -m pip wheel`, or "build", which will run `python -m build --wheel`. + +#### Examples + +!!! tab examples "Environment variables" + + ```yaml + # Switch to using build + CIBW_BUILD_FRONTEND: "build" + + # Ensure pip is used even if the default changes in the future + CIBW_BUILD_FRONTEND: "pip" + ``` + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Switch to using build + build-frontend = "build" + + # Ensure pip is used even if the default changes in the future + build-frontend = "pip" + ``` + + ### `CIBW_ENVIRONMENT` {: #environment} > Set environment variables needed during the build diff --git a/test/conftest.py b/test/conftest.py index ce433b8a0..06cb577c5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -23,6 +23,8 @@ def pytest_collection_modifyitems(config, items) -> None: item.add_marker(skip_emulation) -@pytest.fixture(params=[{"CIBW_PYPA_BUILD": "0"}, {"CIBW_PYPA_BUILD": "1"}], ids=["pip", "pypa"]) +@pytest.fixture( + params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"] +) def build_mode(request) -> Dict[str, str]: - return request.param + return request.param # type: ignore From 33d3a581700882943734444814d9b013bfaa9841 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 22 Jun 2021 10:27:51 -0400 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Joe Rickerby --- cibuildwheel/__main__.py | 2 +- cibuildwheel/linux.py | 2 +- cibuildwheel/macos.py | 2 +- cibuildwheel/windows.py | 2 +- docs/options.md | 11 ++++++++++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 4e5b9a00b..15eedbf45 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -204,7 +204,7 @@ def main() -> None: ) if build_frontend not in BUILD_FRONTENDS: - msg = f"cibuildwheel: Unrecognised build front end '{build_frontend}', only {BUILD_FRONTENDS} supported" + msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only {BUILD_FRONTENDS} supported" print(msg, file=sys.stderr) sys.exit(2) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d2bf56ab9..7088df2c5 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -317,7 +317,7 @@ def troubleshoot(package_dir: Path, error: Exception) -> None: error.cmd[0:4] == ["python", "-m", "pip", "wheel"] or error.cmd[0:3] == ["python", "-m", "build"] ): - # the 'pip wheel' step failed. + # the wheel build step failed print("Checking for common errors...") so_files = list(package_dir.glob("**/*.so")) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 5c94b34d5..b606578c9 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -413,7 +413,7 @@ def build(options: BuildOptions) -> None: ) elif options.build_frontend == "build": config_setting = " ".join(verbosity_flags) - build_env = dict(env) + build_env = env.copy() if options.dependency_constraints: build_env["PIP_CONSTRAINT"] = str( options.dependency_constraints.get_for_python_version(config.version) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 188efd805..428d9faaa 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -310,7 +310,7 @@ def build(options: BuildOptions) -> None: ) elif options.build_frontend == "build": config_setting = " ".join(verbosity_flags) - build_env = dict(env) + build_env = env.copy() if options.dependency_constraints: build_env["PIP_CONSTRAINT"] = str( options.dependency_constraints.get_for_python_version(config.version) diff --git a/docs/options.md b/docs/options.md index 60c427b61..f07a42615 100644 --- a/docs/options.md +++ b/docs/options.md @@ -438,9 +438,18 @@ This option can also be set using the [command-line option](#command-line) `--pr ### `CIBW_BUILD_FRONTEND` {: #build-frontend} > Set the tool to use to build, either "pip" (default for now) or "build" -A selector for the build backend to use. Can either be "pip", which will run +Choose which build backend to use. Can either be "pip", which will run `python -m pip wheel`, or "build", which will run `python -m build --wheel`. +!!! tip + Until v2.0.0, [pip] was the only way to build wheels, and is still the + default. However, we expect that at some point in the future, cibuildwheel + will change the default to [build], in line with the PyPA's recommendation. + If you want to try `build` before this, you can use this option. + +[pip]: https://pip.pypa.io/en/stable/cli/pip_wheel/ +[build]: https://github.com/pypa/build/ + #### Examples !!! tab examples "Environment variables" From b22f47c111321175e5bc97fbcf847ca82ada43bd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 22 Jun 2021 11:15:25 -0400 Subject: [PATCH 4/6] refactor: use literal types for build_frontend --- cibuildwheel/__main__.py | 14 +++++++++----- cibuildwheel/linux.py | 6 ++---- cibuildwheel/macos.py | 9 +++++---- cibuildwheel/util.py | 6 ++++-- cibuildwheel/windows.py | 9 +++++---- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 15eedbf45..a1cb85cd0 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -20,6 +20,7 @@ from cibuildwheel.projectfiles import get_requires_python_str from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never from cibuildwheel.util import ( + BuildFrontend, BuildOptions, BuildSelector, DependencyConstraints, @@ -40,8 +41,6 @@ "pypy_i686", ) -BUILD_FRONTENDS = {"pip", "build"} - def main() -> None: platform: PlatformName @@ -186,7 +185,7 @@ def main() -> None: archs_config_str = args.archs or options("archs", sep=" ") - build_frontend = options("build-frontend", env_plat=False) + build_frontend_str = options("build-frontend", env_plat=False) environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "}) before_all = options("before-all", sep=" && ") before_build = options("before-build", sep=" && ") @@ -203,8 +202,13 @@ def main() -> None: os.environ.get("CIBW_PRERELEASE_PYTHONS", "0") ) - if build_frontend not in BUILD_FRONTENDS: - msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only {BUILD_FRONTENDS} supported" + build_frontend: BuildFrontend + if build_frontend_str == "build": + build_frontend = "build" + elif build_frontend_str == "pip": + build_frontend = "pip" + else: + msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only 'pip' and 'build' are supported" print(msg, file=sys.stderr) sys.exit(2) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 7088df2c5..f22304c05 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -7,7 +7,7 @@ from .architecture import Architecture from .docker_container import DockerContainer from .logger import log -from .typing import PathOrStr +from .typing import PathOrStr, assert_never from .util import ( BuildOptions, BuildSelector, @@ -211,9 +211,7 @@ def build(options: BuildOptions) -> None: env=env, ) else: - raise RuntimeError( - f"build_frontend {options.build_frontend!r} not understood" - ) + assert_never(options.build_frontend) built_wheel = docker.glob(built_wheel_dir, "*.whl")[0] diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index b606578c9..72cce89fe 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -12,8 +12,9 @@ from .architecture import Architecture from .environment import ParsedEnvironment from .logger import log -from .typing import Literal, PathOrStr +from .typing import Literal, PathOrStr, assert_never from .util import ( + BuildFrontend, BuildOptions, BuildSelector, NonPlatformWheelError, @@ -179,7 +180,7 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, - build_frontend: str, + build_frontend: BuildFrontend, ) -> Dict[str, str]: implementation_id = python_configuration.identifier.split("-")[0] @@ -337,7 +338,7 @@ def setup_python( env=env, ) else: - raise RuntimeError(f"build_frontend {build_frontend!r} not understood") + assert_never(build_frontend) return env @@ -432,7 +433,7 @@ def build(options: BuildOptions) -> None: env=build_env, ) else: - raise RuntimeError(f"build_frontend {options.build_frontend!r} not understood") + assert_never(options.build_frontend) built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index e7e5a675a..3fb47ef3e 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -22,12 +22,14 @@ from .architecture import Architecture from .environment import ParsedEnvironment -from .typing import PathOrStr, PlatformName +from .typing import Literal, PathOrStr, PlatformName resources_dir = Path(__file__).parent / "resources" install_certifi_script = resources_dir / "install_certifi.py" +BuildFrontend = Literal["pip", "build"] + def prepare_command(command: str, **kwargs: PathOrStr) -> str: """ @@ -220,7 +222,7 @@ class BuildOptions(NamedTuple): test_requires: List[str] test_extras: str build_verbosity: int - build_frontend: str + build_frontend: BuildFrontend class NonPlatformWheelError(Exception): diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 428d9faaa..07b501dff 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -10,8 +10,9 @@ from .architecture import Architecture from .environment import ParsedEnvironment from .logger import log -from .typing import PathOrStr +from .typing import PathOrStr, assert_never from .util import ( + BuildFrontend, BuildOptions, BuildSelector, NonPlatformWheelError, @@ -115,7 +116,7 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, - build_frontend: str, + build_frontend: BuildFrontend, ) -> Dict[str, str]: nuget = Path("C:\\cibw\\nuget.exe") @@ -236,7 +237,7 @@ def setup_python( env=env, ) else: - raise RuntimeError(f"build_frontend {build_frontend!r} not understood") + assert_never(build_frontend) return env @@ -329,7 +330,7 @@ def build(options: BuildOptions) -> None: env=build_env, ) else: - raise RuntimeError(f"build_frontend {options.build_frontend!r} not understood") + assert_never(options.build_frontend) built_wheel = next(built_wheel_dir.glob("*.whl")) From 6c4d21a6fc10a993ad90ad74dd2a1a76e0fadf29 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 22 Jun 2021 11:18:58 -0400 Subject: [PATCH 5/6] refactor(tests): build_mode -> build_frontend_env --- test/conftest.py | 2 +- test/test_0_basic.py | 4 ++-- test/test_dependency_versions.py | 8 ++++---- test/test_pep518.py | 4 ++-- test/test_troubleshooting.py | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 06cb577c5..6e3293b26 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -26,5 +26,5 @@ def pytest_collection_modifyitems(config, items) -> None: @pytest.fixture( params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"] ) -def build_mode(request) -> Dict[str, str]: +def build_frontend_env(request) -> Dict[str, str]: return request.param # type: ignore diff --git a/test/test_0_basic.py b/test/test_0_basic.py index d012fe06f..8f2cd4bfb 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -18,12 +18,12 @@ ) -def test(tmp_path, build_mode): +def test(tmp_path, build_frontend_env): project_dir = tmp_path / "project" basic_project.generate(project_dir) # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_mode) + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_frontend_env) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 76a2dd866..1090450f3 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -48,7 +48,7 @@ def get_versions_from_constraint_file(constraint_file): @pytest.mark.parametrize("python_version", ["3.6", "3.8", "3.9"]) -def test_pinned_versions(tmp_path, python_version, build_mode): +def test_pinned_versions(tmp_path, python_version, build_frontend_env): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -85,7 +85,7 @@ def test_pinned_versions(tmp_path, python_version, build_mode): add_env={ "CIBW_BUILD": build_pattern, "CIBW_ENVIRONMENT": cibw_environment_option, - **build_mode, + **build_frontend_env, }, ) @@ -108,7 +108,7 @@ def test_pinned_versions(tmp_path, python_version, build_mode): assert set(actual_wheels) == set(expected_wheels) -def test_dependency_constraints_file(tmp_path, build_mode): +def test_dependency_constraints_file(tmp_path, build_frontend_env): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -151,7 +151,7 @@ def test_dependency_constraints_file(tmp_path, build_mode): add_env={ "CIBW_ENVIRONMENT": cibw_environment_option, "CIBW_DEPENDENCY_VERSIONS": str(constraints_file), - **build_mode, + **build_frontend_env, }, ) diff --git a/test/test_pep518.py b/test/test_pep518.py index 33be7d273..128834280 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -32,13 +32,13 @@ """ -def test_pep518(tmp_path, build_mode): +def test_pep518(tmp_path, build_frontend_env): project_dir = tmp_path / "project" basic_project.generate(project_dir) # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_mode) + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_frontend_env) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") diff --git a/test/test_troubleshooting.py b/test/test_troubleshooting.py index e1ccf48d6..6a58ca1fd 100644 --- a/test/test_troubleshooting.py +++ b/test/test_troubleshooting.py @@ -16,7 +16,7 @@ """ -def test_failed_project_with_so_files(tmp_path, capfd, build_mode): +def test_failed_project_with_so_files(tmp_path, capfd, build_frontend_env): if utils.platform != "linux": pytest.skip("this test is only relevant to the linux build") @@ -24,7 +24,7 @@ def test_failed_project_with_so_files(tmp_path, capfd, build_mode): so_file_project.generate(project_dir) with pytest.raises(subprocess.CalledProcessError): - utils.cibuildwheel_run(project_dir, add_env=build_mode) + utils.cibuildwheel_run(project_dir, add_env=build_frontend_env) captured = capfd.readouterr() print("out", captured.out) From a564d32da2b44f833a8bb494dbb94c2702378582 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 22 Jun 2021 18:16:32 -0400 Subject: [PATCH 6/6] Update test/test_before_build.py --- test/test_before_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_before_build.py b/test/test_before_build.py index c451abb62..00906acc1 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -31,7 +31,7 @@ current_path = os.path.realpath(sys.executable).lower() # TODO: This is not valid in an virtual environment - # assert stored_path == current_path, '{0} != {1}'.format(stored_path, current_path) + assert stored_path == current_path, '{0} != {1}'.format(stored_path, current_path) """ ) )