From b0f076b977ea54499d3de28528d47c4f8379ef42 Mon Sep 17 00:00:00 2001 From: mayeut Date: Wed, 2 Oct 2024 20:10:03 +0200 Subject: [PATCH 1/2] feat: use limited API to build PyPI wheels --- pyproject.toml | 2 +- scripts/manylinux/build_on_centos.sh | 44 +++++----------------------- scripts/manylinux/check.sh | 2 +- scripts/osx/build.sh | 14 ++++----- scripts/windows/build.bat | 8 +++++ setup.py | 11 ++++++- 6 files changed, 34 insertions(+), 47 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 85344444..576508f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] # Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel"] # PEP 508 specifications. \ No newline at end of file +requires = ["setuptools"] # PEP 508 specifications. diff --git a/scripts/manylinux/build_on_centos.sh b/scripts/manylinux/build_on_centos.sh index 2df9c2ca..11f562b4 100755 --- a/scripts/manylinux/build_on_centos.sh +++ b/scripts/manylinux/build_on_centos.sh @@ -39,45 +39,15 @@ ${MAIN_PYTHON_BIN}/cmake \ .. make all install -PYTHON_VERSIONS="" -if [[ -z ${BUILD_PYTHON} ]]; then - # Collect all target Python versions. - for PYTHON_BIN in /opt/python/*/bin; do - # H/T: https://stackoverflow.com/a/229606/1068170 - if [[ "${PYTHON_BIN}" == *"39"* ]]; then - PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" - continue - elif [[ "${PYTHON_BIN}" == *"310"* ]]; then - PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" - continue - elif [[ "${PYTHON_BIN}" == *"311"* ]]; then - PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" - continue - elif [[ "${PYTHON_BIN}" == *"312"* ]]; then - PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" - continue - else - echo "Ignoring unsupported version: ${PYTHON_BIN}" - echo "=====================================" - fi - done -else - STRIPPED_PYTHON=$(echo ${BUILD_PYTHON} | sed -e "s/\.//g" | sed -e "s/-dev$//") - for PYTHON_BIN in /opt/python/*/bin; do - if [[ "${PYTHON_BIN}" == *"${STRIPPED_PYTHON}"* ]]; then - PYTHON_VERSIONS="${PYTHON_VERSIONS} ${PYTHON_BIN}" - fi - done -fi -# Build the wheels. +# Build the wheel. +export CRC32C_PURE_PYTHON=0 +export CRC32C_LIMITED_API=1 cd ${REPO_ROOT} -for PYTHON_BIN in ${PYTHON_VERSIONS}; do - ${PYTHON_BIN}/python -m pip install --upgrade pip - ${PYTHON_BIN}/python -m pip install \ - --requirement ${REPO_ROOT}/scripts/dev-requirements.txt - ${PYTHON_BIN}/python -m pip wheel . --wheel-dir dist_wheels/ -done +${MAIN_PYTHON_BIN}/python -m pip install --upgrade pip +${MAIN_PYTHON_BIN}/python -m pip install \ + --requirement ${REPO_ROOT}/scripts/dev-requirements.txt +${MAIN_PYTHON_BIN}/python -m pip wheel . --wheel-dir dist_wheels/ # Bundle external shared libraries into the wheels for whl in dist_wheels/google_crc32c*.whl; do diff --git a/scripts/manylinux/check.sh b/scripts/manylinux/check.sh index f5f38150..ce8f9ae7 100755 --- a/scripts/manylinux/check.sh +++ b/scripts/manylinux/check.sh @@ -48,7 +48,7 @@ for PYTHON_VERSION in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do ${PYTHON} -m venv venv # Install the wheel. - WHEEL_FILE="wheels/google_crc32c-${PACKAGE_VERSION}-cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + WHEEL_FILE="wheels/google_crc32c-${PACKAGE_VERSION}-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" venv/bin/pip install ${WHEEL_FILE} # Verify that the module is installed and peek at contents. diff --git a/scripts/osx/build.sh b/scripts/osx/build.sh index 326ba92d..bf250022 100755 --- a/scripts/osx/build.sh +++ b/scripts/osx/build.sh @@ -42,14 +42,14 @@ git submodule update --init --recursive ${OSX_DIR}/build_c_lib.sh +export CRC32C_PURE_PYTHON=0 +export CRC32C_LIMITED_API=1 SUPPORTED_PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12") - -for PYTHON_VERSION in ${SUPPORTED_PYTHON_VERSIONS[@]}; do - echo "Build wheel for Python ${PYTHON_VERSION}" - export PY_BIN=$PYTHON_VERSION - export PY_TAG="cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}" - . /${OSX_DIR}/build_python_wheel.sh -done +PYTHON_VERSION=${SUPPORTED_PYTHON_VERSIONS[0]}; +echo "Build wheel for Python ${PYTHON_VERSION}" +export PY_BIN=$PYTHON_VERSION +export PY_TAG="cp${PYTHON_VERSION//.}-abi3" +. /${OSX_DIR}/build_python_wheel.sh # Clean up. rm -fr ${CRC32C_INSTALL_PREFIX} diff --git a/scripts/windows/build.bat b/scripts/windows/build.bat index 591834a6..b3a27f49 100644 --- a/scripts/windows/build.bat +++ b/scripts/windows/build.bat @@ -18,6 +18,9 @@ setlocal ENABLEDELAYEDEXPANSION set CMAKE_GENERATOR="Visual Studio 17 2022" set CONFIGURATION=RelWithDebInfo set CRC32C_INSTALL_PREFIX=%cd%\build\%CONFIGURATION% +set CRC32C_PURE_PYTHON=0 +set CRC32C_LIMITED_API=1 +set BUILT=0 @rem Iterate through supported Python versions. @rem Unfortunately pyenv for Windows has an out-of-date versions list. Choco's @@ -33,6 +36,8 @@ FOR %%P IN (3.9, 3.10, 3.11, 3.12) DO ( py -%%P-64 -m pip install --upgrade pip + if "%BUILT%"=="1" (goto test) + echo "Installing cmake for Python %%P" py -%%P-64 -m pip install cmake @@ -73,6 +78,9 @@ FOR %%P IN (3.9, 3.10, 3.11, 3.12) DO ( echo "Building Wheel" py -%%P-64 -m pip wheel . --wheel-dir wheels/ + set BUILT=1 + +:test echo "Built wheel, now running tests." call %~dp0/test.bat %%P || goto :error diff --git a/setup.py b/setup.py index 5c304b20..d039f4c3 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,8 @@ CRC32C_PURE_PYTHON_EXPLICIT = "CRC32C_PURE_PYTHON" in os.environ _FALSE_OPTIONS = ("0", "false", "no", "False", "No", None) CRC32C_PURE_PYTHON = os.getenv("CRC32C_PURE_PYTHON") not in _FALSE_OPTIONS +# Wether or not we want to build using limited API (ABI3) +CRC32C_LIMITED_API = os.getenv("CRC32C_LIMITED_API") not in _FALSE_OPTIONS def copy_dll(build_lib): @@ -90,6 +92,13 @@ def build_c_extension(): print("#### using global install of 'crc32c'") kwargs = {} + if CRC32C_LIMITED_API: + kwargs["define_macros"] = [("Py_LIMITED_API", "0x03090000")] + kwargs["py_limited_api"] = True + options = {"bdist_wheel": {"py_limited_api": "cp39"}} + else: + options = {} + module_path = os.path.join("src", "google_crc32c", "_crc32c.c") sources=[os.path.normcase(module_path)] print(f"##### sources: {sources}") @@ -106,7 +115,7 @@ def build_c_extension(): package_dir={"": "src"}, ext_modules=[module], cmdclass={"build_ext": BuildExtWithDLL}, - install_requires=["importlib_resources>=1.3 ; python_version < '3.9' and os_name == 'nt'"], + options=options, ) From 71c878b73b350743fd34f432a23f6691b7277cde Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 8 Mar 2025 07:53:37 +0100 Subject: [PATCH 2/2] address review comments --- scripts/manylinux/check.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/manylinux/check.sh b/scripts/manylinux/check.sh index ce8f9ae7..f8ee32b4 100755 --- a/scripts/manylinux/check.sh +++ b/scripts/manylinux/check.sh @@ -39,6 +39,9 @@ install_python_pyenv() { SUPPORTED_PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12") +PYTHON_VERSION_MIN=${SUPPORTED_PYTHON_VERSIONS[0]} +PY_TAG="cp${PYTHON_VERSION_MIN//.}-abi3" + for PYTHON_VERSION in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do PYTHON=python${PYTHON_VERSION} install_python_pyenv ${PYTHON_VERSION} @@ -48,7 +51,7 @@ for PYTHON_VERSION in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do ${PYTHON} -m venv venv # Install the wheel. - WHEEL_FILE="wheels/google_crc32c-${PACKAGE_VERSION}-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + WHEEL_FILE="wheels/google_crc32c-${PACKAGE_VERSION}-${PY_TAG}-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" venv/bin/pip install ${WHEEL_FILE} # Verify that the module is installed and peek at contents.