Skip to content

refactor: identifiers in a file #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
henryiii merged 8 commits into from
Jan 9, 2021
Merged

refactor: identifiers in a file #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
97ddab5
refactor: identifiers in a file
henryiii Jan 2, 2021
b1f1f23
chore: more typing
henryiii Jan 6, 2021
9bb8c95
refactor: PlatStr -> PlatformName
henryiii Jan 9, 2021
b54f665
refactor: load in file, use importlib.resources
henryiii Jan 9, 2021
9090072
refactor: change toml structure
henryiii Jan 9, 2021
67f59dd
refactor: use common Traversable
henryiii Jan 9, 2021
1d91408
fix: ensure reasonable version of importlib_resources
henryiii Jan 9, 2021
4674245
style: fix extra line
henryiii Jan 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions cibuildwheel/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import cibuildwheel.macos
import cibuildwheel.windows
from cibuildwheel.environment import EnvironmentParseError, parse_environment
from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
from cibuildwheel.util import (
Architecture,
BuildOptions,
Expand Down Expand Up @@ -46,6 +47,8 @@ def get_option_from_environment(option_name: str, platform: Optional[str] = None


def main() -> None:
platform: PlatformName

parser = argparse.ArgumentParser(
description='Build wheels for all the platforms.',
epilog='''
Expand Down Expand Up @@ -120,15 +123,21 @@ def main() -> None:
file=sys.stderr)
exit(2)

if platform not in PLATFORMS:
print(f'cibuildwheel: Unsupported platform: {platform}', file=sys.stderr)
exit(2)

package_dir = Path(args.package_dir)
output_dir = Path(args.output_dir)

if platform == 'linux':
repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
elif platform == 'macos':
repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'
else:
elif platform == 'windows':
repair_command_default = ''
else:
assert_never(platform)

build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
Expand Down Expand Up @@ -248,8 +257,7 @@ def main() -> None:
elif platform == 'macos':
cibuildwheel.macos.build(build_options)
else:
print(f'cibuildwheel: Unsupported platform: {platform}', file=sys.stderr)
exit(2)
assert_never(platform)


def detect_obsolete_options() -> None:
Expand Down Expand Up @@ -302,6 +310,7 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None:
def print_build_identifiers(
platform: str, build_selector: BuildSelector, architectures: Set[Architecture]
) -> None:

python_configurations: List[Any] = []
if platform == 'linux':
python_configurations = cibuildwheel.linux.get_python_configurations(build_selector, architectures)
Expand Down
47 changes: 10 additions & 37 deletions cibuildwheel/linux.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
allowed_architectures_check,
get_build_verbosity_extra_flags,
prepare_command,
read_python_configs,
resources_dir,
)


Expand All @@ -29,42 +31,13 @@ def path(self) -> PurePath:


def get_python_configurations(
build_selector: BuildSelector, architectures: Set[Architecture]
build_selector: BuildSelector,
architectures: Set[Architecture]
) -> List[PythonConfiguration]:
python_configurations = [
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.9', identifier='cp39-manylinux_x86_64', path_str='/opt/python/cp39-cp39'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27mu'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.9', identifier='cp39-manylinux_i686', path_str='/opt/python/cp39-cp39'),
PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path_str='/opt/python/pp27-pypy_73'),
PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path_str='/opt/python/pp36-pypy36_pp73'),
PythonConfiguration(version='3.7', identifier='pp37-manylinux_x86_64', path_str='/opt/python/pp37-pypy37_pp73'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.9', identifier='cp39-manylinux_aarch64', path_str='/opt/python/cp39-cp39'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.9', identifier='cp39-manylinux_ppc64le', path_str='/opt/python/cp39-cp39'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'),
]

full_python_configs = read_python_configs('linux')

python_configurations = [PythonConfiguration(**item) for item in full_python_configs]

# return all configurations whose arch is in our `architectures` set,
# and match the build/skip rules
Expand All @@ -76,7 +49,7 @@ def get_python_configurations(


def build(options: BuildOptions) -> None:
allowed_architectures_check("linux", options)
allowed_architectures_check('linux', options)

try:
subprocess.check_output(['docker', '--version'])
Expand Down Expand Up @@ -136,7 +109,7 @@ def build(options: BuildOptions) -> None:
if config.identifier.startswith("pp"):
# Patch PyPy to make sure headers get installed into a venv
patch_version = '_27' if config.version == '2.7' else ''
patch_path = Path(__file__).absolute().parent / 'resources' / f'pypy_venv{patch_version}.patch'
patch_path = resources_dir / f'pypy_venv{patch_version}.patch'
patch_docker_path = PurePath('/pypy_venv.patch')
docker.copy_into(patch_path, patch_docker_path)
try:
Expand Down
24 changes: 9 additions & 15 deletions cibuildwheel/macos.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
get_pip_script,
install_certifi_script,
prepare_command,
read_python_configs,
resources_dir,
)


Expand All @@ -39,20 +41,12 @@ class PythonConfiguration(NamedTuple):
url: str


def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]:
python_configurations = [
# CPython
PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'),
PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg'),
PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg'),
PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg'),
# PyPy
PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'),
PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'),
PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'),
]
def get_python_configurations(
build_selector: BuildSelector) -> List[PythonConfiguration]:

full_python_configs = read_python_configs('macos')

python_configurations = [PythonConfiguration(**item) for item in full_python_configs]

# skip builds as required
return [c for c in python_configurations if build_selector(c.identifier)]
Expand Down Expand Up @@ -115,7 +109,7 @@ def install_pypy(version: str, url: str) -> Path:
call(['tar', '-C', '/tmp', '-xf', downloaded_tar_bz2])
# Patch PyPy to make sure headers get installed into a venv
patch_version = '_27' if version == '2.7' else ''
patch_path = Path(__file__).absolute().parent / 'resources' / f'pypy_venv{patch_version}.patch'
patch_path = resources_dir / f'pypy_venv{patch_version}.patch'
call(['patch', '--force', '-p1', '-d', installation_path, '-i', patch_path])

installation_bin_path = installation_path / 'bin'
Expand Down
67 changes: 67 additions & 0 deletions cibuildwheel/resources/build-platforms.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
[linux]
python_configurations = [
{ identifier = "cp27-manylinux_x86_64", version = "2.7", path_str = "/opt/python/cp27-cp27m" },
{ identifier = "cp27-manylinux_x86_64", version = "2.7", path_str = "/opt/python/cp27-cp27mu" },
{ identifier = "cp35-manylinux_x86_64", version = "3.5", path_str = "/opt/python/cp35-cp35m" },
{ identifier = "cp36-manylinux_x86_64", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
{ identifier = "cp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
{ identifier = "cp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/cp38-cp38" },
{ identifier = "cp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/cp39-cp39" },
{ identifier = "cp27-manylinux_i686", version = "2.7", path_str = "/opt/python/cp27-cp27m" },
{ identifier = "cp27-manylinux_i686", version = "2.7", path_str = "/opt/python/cp27-cp27mu" },
{ identifier = "cp35-manylinux_i686", version = "3.5", path_str = "/opt/python/cp35-cp35m" },
{ identifier = "cp36-manylinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
{ identifier = "cp37-manylinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
{ identifier = "cp38-manylinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" },
{ identifier = "cp39-manylinux_i686", version = "3.9", path_str = "/opt/python/cp39-cp39" },
{ identifier = "pp27-manylinux_x86_64", version = "2.7", path_str = "/opt/python/pp27-pypy_73" },
{ identifier = "pp36-manylinux_x86_64", version = "3.6", path_str = "/opt/python/pp36-pypy36_pp73" },
{ identifier = "pp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" },
{ identifier = "cp35-manylinux_aarch64", version = "3.5", path_str = "/opt/python/cp35-cp35m" },
{ identifier = "cp36-manylinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
{ identifier = "cp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
{ identifier = "cp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" },
{ identifier = "cp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" },
{ identifier = "cp35-manylinux_ppc64le", version = "3.5", path_str = "/opt/python/cp35-cp35m" },
{ identifier = "cp36-manylinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
{ identifier = "cp37-manylinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
{ identifier = "cp38-manylinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" },
{ identifier = "cp39-manylinux_ppc64le", version = "3.9", path_str = "/opt/python/cp39-cp39" },
{ identifier = "cp35-manylinux_s390x", version = "3.5", path_str = "/opt/python/cp35-cp35m" },
{ identifier = "cp36-manylinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
{ identifier = "cp37-manylinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
{ identifier = "cp38-manylinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" },
{ identifier = "cp39-manylinux_s390x", version = "3.9", path_str = "/opt/python/cp39-cp39" },
]

[macos]
python_configurations = [
{ identifier = "cp27-macosx_x86_64", version = "2.7", url = "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg" },
{ identifier = "cp35-macosx_x86_64", version = "3.5", url = "https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg" },
{ identifier = "cp36-macosx_x86_64", version = "3.6", url = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg" },
{ identifier = "cp37-macosx_x86_64", version = "3.7", url = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg" },
{ identifier = "cp38-macosx_x86_64", version = "3.8", url = "https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg" },
{ identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg" },
{ identifier = "pp27-macosx_x86_64", version = "2.7", url = "https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2" },
{ identifier = "pp36-macosx_x86_64", version = "3.6", url = "https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2" },
{ identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2" },
]

[windows]
python_configurations = [
{ identifier = "cp27-win32", version = "2.7.18", arch = "32" },
{ identifier = "cp27-win_amd64", version = "2.7.18", arch = "64" },
{ identifier = "cp35-win32", version = "3.5.4", arch = "32" },
{ identifier = "cp35-win_amd64", version = "3.5.4", arch = "64" },
{ identifier = "cp36-win32", version = "3.6.8", arch = "32" },
{ identifier = "cp36-win_amd64", version = "3.6.8", arch = "64" },
{ identifier = "cp37-win32", version = "3.7.9", arch = "32" },
{ identifier = "cp37-win_amd64", version = "3.7.9", arch = "64" },
{ identifier = "cp38-win32", version = "3.8.7", arch = "32" },
{ identifier = "cp38-win_amd64", version = "3.8.7", arch = "64" },
{ identifier = "cp39-win32", version = "3.9.1", arch = "32" },
{ identifier = "cp39-win_amd64", version = "3.9.1", arch = "64" },
{ identifier = "pp27-win32", version = "2.7", arch = "32", url = "https://downloads.python.org/pypy/pypy2.7-v7.3.3-win32.zip" },
{ identifier = "pp36-win32", version = "3.6", arch = "32", url = "https://downloads.python.org/pypy/pypy3.6-v7.3.3-win32.zip" },
{ identifier = "pp37-win32", version = "3.7", arch = "32", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.3-win32.zip" },
]
17 changes: 16 additions & 1 deletion cibuildwheel/typing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,25 @@
import os
import subprocess
from typing import TYPE_CHECKING, Union
import sys
from typing import TYPE_CHECKING, NoReturn, Set, Union

if sys.version_info < (3, 8):
from typing_extensions import Final, Literal
else:
from typing import Final, Literal


if TYPE_CHECKING:
PopenBytes = subprocess.Popen[bytes]
PathOrStr = Union[str, os.PathLike[str]]
else:
PopenBytes = subprocess.Popen
PathOrStr = Union[str, "os.PathLike[str]"]


PlatformName = Literal["linux", "macos", "windows"]
PLATFORMS: Final[Set[PlatformName]] = {"linux", "macos", "windows"}


def assert_never(value: NoReturn) -> NoReturn:
assert False, f'Unhandled value: {value} ({type(value).__name__})' # noqa: B011
32 changes: 20 additions & 12 deletions cibuildwheel/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,20 @@
from typing import Dict, List, NamedTuple, Optional, Set

import certifi
import toml

from .environment import ParsedEnvironment
from .typing import PathOrStr
from .typing import PathOrStr, PlatformName

if sys.version_info < (3, 8):
from typing_extensions import Literal
if sys.version_info < (3, 9):
from importlib_resources import files
else:
from typing import Literal
from importlib.resources import files


resources_dir = files('cibuildwheel') / 'resources'
Copy link
Contributor

@joerick joerick Jan 9, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity, what does importlib.resources.files do that Path(__file__).parent doesn't? It can't be .egg compatibility, can it? :P

Copy link
Contributor Author

@henryiii henryiii Jan 9, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It supports directories that are zip files (see https://github.com/scikit-hep/particle/releases/tag/v0.14.0 - that .pyz file runs directly without installing, python3 particle.pyz just works), and namespace packages that are split across directories.

Though, however, I'm using it here because it's more elegant and the "right' thing to do. We are accessing a resource.

PS: There's a few tricks when running from a zip file, you can open the path or access the filename, but cannot use the file name to open the file without a little workaround which I didn't attempt to implement.

get_pip_script = resources_dir / 'get-pip.py'
install_certifi_script = resources_dir / "install_certifi.py"


def prepare_command(command: str, **kwargs: PathOrStr) -> str:
Expand All @@ -42,6 +48,13 @@ def get_build_verbosity_extra_flags(level: int) -> List[str]:
return []


def read_python_configs(config: PlatformName) -> List[Dict[str, str]]:
input_file = resources_dir / 'build-platforms.toml'
loaded_file = toml.load(input_file)
results: List[Dict[str, str]] = list(loaded_file[config]['python_configurations'])
return results


class BuildSelector:
def __init__(self, build_config: str, skip_config: str):
self.build_patterns = build_config.split()
Expand Down Expand Up @@ -151,7 +164,7 @@ def __lt__(self, other: "Architecture") -> bool:
return self.value < other.value

@staticmethod
def parse_config(config: str, platform: str) -> 'Set[Architecture]':
def parse_config(config: str, platform: PlatformName) -> 'Set[Architecture]':
result = set()
for arch_str in re.split(r'[\s,]+', config):
if arch_str == 'auto':
Expand All @@ -161,7 +174,7 @@ def parse_config(config: str, platform: str) -> 'Set[Architecture]':
return result

@staticmethod
def auto_archs(platform: str) -> 'Set[Architecture]':
def auto_archs(platform: PlatformName) -> 'Set[Architecture]':
native_architecture = Architecture(platform_module.machine())
result = {native_architecture}
if platform == 'linux' and native_architecture == Architecture.x86_64:
Expand Down Expand Up @@ -190,11 +203,6 @@ class BuildOptions(NamedTuple):
build_verbosity: int


resources_dir = Path(__file__).resolve().parent / 'resources'
get_pip_script = resources_dir / 'get-pip.py'
install_certifi_script = resources_dir / "install_certifi.py"


class NonPlatformWheelError(Exception):
def __init__(self) -> None:
message = textwrap.dedent('''
Expand Down Expand Up @@ -255,7 +263,7 @@ def detect_ci_provider() -> Optional[CIProvider]:


def allowed_architectures_check(
name: Literal['linux', 'macos', 'windows'],
name: PlatformName,
options: BuildOptions,
) -> None:

Expand Down
Loading