diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 1d3ce40c4..a1cb85cd0 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -20,6 +20,7 @@ from cibuildwheel.projectfiles import get_requires_python_str from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never from cibuildwheel.util import ( + BuildFrontend, BuildOptions, BuildSelector, DependencyConstraints, @@ -184,6 +185,7 @@ def main() -> None: archs_config_str = args.archs or options("archs", sep=" ") + build_frontend_str = options("build-frontend", env_plat=False) environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "}) before_all = options("before-all", sep=" && ") before_build = options("before-build", sep=" && ") @@ -200,6 +202,16 @@ def main() -> None: os.environ.get("CIBW_PRERELEASE_PYTHONS", "0") ) + build_frontend: BuildFrontend + if build_frontend_str == "build": + build_frontend = "build" + elif build_frontend_str == "pip": + build_frontend = "pip" + else: + msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only 'pip' and 'build' are supported" + print(msg, file=sys.stderr) + sys.exit(2) + package_files = {"setup.py", "setup.cfg", "pyproject.toml"} if not any(package_dir.joinpath(name).exists() for name in package_files): @@ -308,6 +320,7 @@ def main() -> None: environment=environment, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images or None, + build_frontend=build_frontend, ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index faeff20b4..f22304c05 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -7,7 +7,7 @@ from .architecture import Architecture from .docker_container import DockerContainer from .logger import log -from .typing import PathOrStr +from .typing import PathOrStr, assert_never from .util import ( BuildOptions, BuildSelector, @@ -145,7 +145,7 @@ def build(options: BuildOptions) -> None: env, executor=docker.environment_executor ) - # check config python and pip are still on PATH + # check config python is still on PATH which_python = docker.call( ["which", "python"], env=env, capture_output=True ).strip() @@ -180,18 +180,38 @@ def build(options: BuildOptions) -> None: docker.call(["rm", "-rf", built_wheel_dir]) docker.call(["mkdir", "-p", built_wheel_dir]) - docker.call( - [ - "pip", - "wheel", - container_package_dir, - "--wheel-dir", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.build_frontend == "pip": + docker.call( + [ + "python", + "-m", + "pip", + "wheel", + container_package_dir, + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, + ], + env=env, + ) + elif options.build_frontend == "build": + config_setting = " ".join(verbosity_flags) + docker.call( + [ + "python", + "-m", + "build", + container_package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=env, + ) + else: + assert_never(options.build_frontend) built_wheel = docker.glob(built_wheel_dir, "*.whl")[0] @@ -291,8 +311,11 @@ def build(options: BuildOptions) -> None: def troubleshoot(package_dir: Path, error: Exception) -> None: - if isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ["pip", "wheel"]: - # the 'pip wheel' step failed. + if isinstance(error, subprocess.CalledProcessError) and ( + error.cmd[0:4] == ["python", "-m", "pip", "wheel"] + or error.cmd[0:3] == ["python", "-m", "build"] + ): + # the wheel build step failed print("Checking for common errors...") so_files = list(package_dir.glob("**/*.so")) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index cbaeae61d..72cce89fe 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -12,13 +12,15 @@ from .architecture import Architecture from .environment import ParsedEnvironment from .logger import log -from .typing import Literal, PathOrStr +from .typing import Literal, PathOrStr, assert_never from .util import ( + BuildFrontend, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, + get_pip_version, install_certifi_script, prepare_command, read_python_configs, @@ -178,7 +180,9 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, + build_frontend: BuildFrontend, ) -> Dict[str, str]: + implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") @@ -308,18 +312,33 @@ def setup_python( env.setdefault("SDKROOT", arm64_compatible_sdks[0]) log.step("Installing build tools...") - call( - [ - "pip", - "install", - "--upgrade", - "setuptools", - "wheel", - "delocate", - *dependency_constraint_flags, - ], - env=env, - ) + if build_frontend == "pip": + call( + [ + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + "delocate", + *dependency_constraint_flags, + ], + env=env, + ) + elif build_frontend == "build": + call( + [ + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv]", + *dependency_constraint_flags, + ], + env=env, + ) + else: + assert_never(build_frontend) return env @@ -356,7 +375,12 @@ def build(options: BuildOptions) -> None: options.dependency_constraints.get_for_python_version(config.version), ] - env = setup_python(config, dependency_constraint_flags, options.environment) + env = setup_python( + config, + dependency_constraint_flags, + options.environment, + options.build_frontend, + ) if options.before_build: log.step("Running before_build...") @@ -370,20 +394,46 @@ def build(options: BuildOptions) -> None: shutil.rmtree(built_wheel_dir) built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "pip", - "wheel", - options.package_dir.resolve(), - "--wheel-dir", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.build_frontend == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *verbosity_flags, + ], + env=env, + ) + elif options.build_frontend == "build": + config_setting = " ".join(verbosity_flags) + build_env = env.copy() + if options.dependency_constraints: + build_env["PIP_CONSTRAINT"] = str( + options.dependency_constraints.get_for_python_version(config.version) + ) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + [ + "python", + "-m", + "build", + options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=build_env, + ) + else: + assert_never(options.build_frontend) built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index a9ecff021..5bf179c89 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -4,6 +4,7 @@ skip = "" test-skip = "" archs = ["auto"] +build-frontend = "pip" dependency-versions = "pinned" environment = {} build-verbosity = "" diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 5d5c47683..3fb47ef3e 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -4,6 +4,8 @@ import os import re import ssl +import subprocess +import sys import textwrap import time import urllib.request @@ -20,12 +22,14 @@ from .architecture import Architecture from .environment import ParsedEnvironment -from .typing import PathOrStr, PlatformName +from .typing import Literal, PathOrStr, PlatformName resources_dir = Path(__file__).parent / "resources" install_certifi_script = resources_dir / "install_certifi.py" +BuildFrontend = Literal["pip", "build"] + def prepare_command(command: str, **kwargs: PathOrStr) -> str: """ @@ -218,6 +222,7 @@ class BuildOptions(NamedTuple): test_requires: List[str] test_extras: str build_verbosity: int + build_frontend: BuildFrontend class NonPlatformWheelError(Exception): @@ -300,3 +305,18 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]: s = time.time() - start_time m = s / 60 print(msg.format(n=n, s=s, m=m), *sorted(f" {f.name}" for f in new_contents), sep="\n") + + +def get_pip_version(env: Dict[str, str]) -> str: + # we use shell=True here for windows, even though we don't need a shell due to a bug + # https://bugs.python.org/issue8557 + shell = sys.platform.startswith("win") + versions_output_text = subprocess.check_output( + ["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env + ) + (pip_version,) = ( + version[5:] + for version in versions_output_text.strip().splitlines() + if version.startswith("pip==") + ) + return pip_version diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 9ce8e309f..07b501dff 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -10,13 +10,15 @@ from .architecture import Architecture from .environment import ParsedEnvironment from .logger import log -from .typing import PathOrStr +from .typing import PathOrStr, assert_never from .util import ( + BuildFrontend, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, + get_pip_version, prepare_command, read_python_configs, ) @@ -114,7 +116,9 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, + build_frontend: BuildFrontend, ) -> Dict[str, str]: + nuget = Path("C:\\cibw\\nuget.exe") if not nuget.exists(): log.step("Downloading nuget...") @@ -214,10 +218,26 @@ def setup_python( sys.exit(1) call(["pip", "--version"], env=env) - call( - ["pip", "install", "--upgrade", "setuptools", "wheel", *dependency_constraint_flags], - env=env, - ) + + if build_frontend == "pip": + call( + [ + "pip", + "install", + "--upgrade", + "setuptools", + "wheel", + *dependency_constraint_flags, + ], + env=env, + ) + elif build_frontend == "build": + call( + ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags], + env=env, + ) + else: + assert_never(build_frontend) return env @@ -251,7 +271,12 @@ def build(options: BuildOptions) -> None: ] # install Python - env = setup_python(config, dependency_constraint_flags, options.environment) + env = setup_python( + config, + dependency_constraint_flags, + options.environment, + options.build_frontend, + ) # run the before_build command if options.before_build: @@ -265,20 +290,47 @@ def build(options: BuildOptions) -> None: if built_wheel_dir.exists(): shutil.rmtree(built_wheel_dir) built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - [ - "pip", - "wheel", - options.package_dir.resolve(), - "-w", - built_wheel_dir, - "--no-deps", - *get_build_verbosity_extra_flags(options.build_verbosity), - ], - env=env, - ) + + verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity) + + if options.build_frontend == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + [ + "python", + "-m", + "pip", + "wheel", + options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *get_build_verbosity_extra_flags(options.build_verbosity), + ], + env=env, + ) + elif options.build_frontend == "build": + config_setting = " ".join(verbosity_flags) + build_env = env.copy() + if options.dependency_constraints: + build_env["PIP_CONSTRAINT"] = str( + options.dependency_constraints.get_for_python_version(config.version) + ) + build_env["VIRTUALENV_PIP"] = get_pip_version(env) + call( + [ + "python", + "-m", + "build", + options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + f"--config-setting={config_setting}", + ], + env=build_env, + ) + else: + assert_never(options.build_frontend) built_wheel = next(built_wheel_dir.glob("*.whl")) diff --git a/docs/options.md b/docs/options.md index 832a8fc4d..f07a42615 100644 --- a/docs/options.md +++ b/docs/options.md @@ -435,6 +435,45 @@ This option can also be set using the [command-line option](#command-line) `--pr ## Build customization +### `CIBW_BUILD_FRONTEND` {: #build-frontend} +> Set the tool to use to build, either "pip" (default for now) or "build" + +Choose which build backend to use. Can either be "pip", which will run +`python -m pip wheel`, or "build", which will run `python -m build --wheel`. + +!!! tip + Until v2.0.0, [pip] was the only way to build wheels, and is still the + default. However, we expect that at some point in the future, cibuildwheel + will change the default to [build], in line with the PyPA's recommendation. + If you want to try `build` before this, you can use this option. + +[pip]: https://pip.pypa.io/en/stable/cli/pip_wheel/ +[build]: https://github.com/pypa/build/ + +#### Examples + +!!! tab examples "Environment variables" + + ```yaml + # Switch to using build + CIBW_BUILD_FRONTEND: "build" + + # Ensure pip is used even if the default changes in the future + CIBW_BUILD_FRONTEND: "pip" + ``` + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Switch to using build + build-frontend = "build" + + # Ensure pip is used even if the default changes in the future + build-frontend = "pip" + ``` + + ### `CIBW_ENVIRONMENT` {: #environment} > Set environment variables needed during the build diff --git a/test/conftest.py b/test/conftest.py index 410c07076..6e3293b26 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,7 +1,9 @@ +from typing import Dict + import pytest -def pytest_addoption(parser): +def pytest_addoption(parser) -> None: parser.addoption( "--run-emulation", action="store_true", default=False, help="run emulation tests" ) @@ -11,7 +13,7 @@ def pytest_configure(config): config.addinivalue_line("markers", "emulation: mark test requiring qemu binfmt_misc to run") -def pytest_collection_modifyitems(config, items): +def pytest_collection_modifyitems(config, items) -> None: if config.getoption("--run-emulation"): # --run-emulation given in cli: do not skip emulation tests return @@ -19,3 +21,10 @@ def pytest_collection_modifyitems(config, items): for item in items: if "emulation" in item.keywords: item.add_marker(skip_emulation) + + +@pytest.fixture( + params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"] +) +def build_frontend_env(request) -> Dict[str, str]: + return request.param # type: ignore diff --git a/test/test_0_basic.py b/test/test_0_basic.py index a84532261..8f2cd4bfb 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -18,12 +18,12 @@ ) -def test(tmp_path): +def test(tmp_path, build_frontend_env): project_dir = tmp_path / "project" basic_project.generate(project_dir) # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir) + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_frontend_env) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") diff --git a/test/test_before_build.py b/test/test_before_build.py index f84bd8007..00906acc1 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -25,8 +25,13 @@ stored_executable = f.read() print('stored_executable', stored_executable) print('sys.executable', sys.executable) + # windows/mac are case insensitive - assert os.path.realpath(stored_executable).lower() == os.path.realpath(sys.executable).lower() + stored_path = os.path.realpath(stored_executable).lower() + current_path = os.path.realpath(sys.executable).lower() + + # TODO: This is not valid in an virtual environment + assert stored_path == current_path, '{0} != {1}'.format(stored_path, current_path) """ ) ) diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index dc1f0b50c..1090450f3 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -48,7 +48,7 @@ def get_versions_from_constraint_file(constraint_file): @pytest.mark.parametrize("python_version", ["3.6", "3.8", "3.9"]) -def test_pinned_versions(tmp_path, python_version): +def test_pinned_versions(tmp_path, python_version, build_frontend_env): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -85,6 +85,7 @@ def test_pinned_versions(tmp_path, python_version): add_env={ "CIBW_BUILD": build_pattern, "CIBW_ENVIRONMENT": cibw_environment_option, + **build_frontend_env, }, ) @@ -107,7 +108,7 @@ def test_pinned_versions(tmp_path, python_version): assert set(actual_wheels) == set(expected_wheels) -def test_dependency_constraints_file(tmp_path): +def test_dependency_constraints_file(tmp_path, build_frontend_env): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -118,7 +119,7 @@ def test_dependency_constraints_file(tmp_path): "pip": "20.0.2", "setuptools": "53.0.0", "wheel": "0.34.2", - "virtualenv": "20.0.10", + "virtualenv": "20.0.35", } constraints_file = tmp_path / "constraints.txt" @@ -129,6 +130,7 @@ def test_dependency_constraints_file(tmp_path): setuptools=={setuptools} wheel=={wheel} virtualenv=={virtualenv} + importlib-metadata<3,>=0.12; python_version < "3.8" """.format( **tool_versions ) @@ -149,6 +151,7 @@ def test_dependency_constraints_file(tmp_path): add_env={ "CIBW_ENVIRONMENT": cibw_environment_option, "CIBW_DEPENDENCY_VERSIONS": str(constraints_file), + **build_frontend_env, }, ) diff --git a/test/test_pep518.py b/test/test_pep518.py index 921024f55..128834280 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -1,4 +1,3 @@ -import os import textwrap from . import test_projects, utils @@ -33,13 +32,13 @@ """ -def test_pep518(tmp_path): +def test_pep518(tmp_path, build_frontend_env): project_dir = tmp_path / "project" basic_project.generate(project_dir) # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir) + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_frontend_env) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") @@ -50,4 +49,12 @@ def test_pep518(tmp_path): assert not (project_dir / "42").exists() assert not (project_dir / "4.1.2").exists() - assert len(os.listdir(project_dir)) == len(basic_project.files) + # pypa/build creates a "build" folder & a "*.egg-info" folder for the wheel being built, + # this should be harmless so remove them + contents = [ + item + for item in project_dir.iterdir() + if item.name != "build" and not item.name.endswith(".egg-info") + ] + + assert len(contents) == len(basic_project.files) diff --git a/test/test_troubleshooting.py b/test/test_troubleshooting.py index e1d2cf8d5..6a58ca1fd 100644 --- a/test/test_troubleshooting.py +++ b/test/test_troubleshooting.py @@ -16,7 +16,7 @@ """ -def test_failed_project_with_so_files(tmp_path, capfd): +def test_failed_project_with_so_files(tmp_path, capfd, build_frontend_env): if utils.platform != "linux": pytest.skip("this test is only relevant to the linux build") @@ -24,7 +24,7 @@ def test_failed_project_with_so_files(tmp_path, capfd): so_file_project.generate(project_dir) with pytest.raises(subprocess.CalledProcessError): - utils.cibuildwheel_run(project_dir) + utils.cibuildwheel_run(project_dir, add_env=build_frontend_env) captured = capfd.readouterr() print("out", captured.out)