From 2ae389d670ec22b9a84e4a9d785a94d2ea03a4bc Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 11 Sep 2024 02:17:57 +0530
Subject: [PATCH 01/51] =?UTF-8?q?Fix=20a=20typo:=20pyoodide=20=E2=9E=A1?=
=?UTF-8?q?=EF=B8=8F=20pyodide?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
test/test_emscripten.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/test_emscripten.py b/test/test_emscripten.py
index 97ceb7f9f..aa7750bc5 100644
--- a/test/test_emscripten.py
+++ b/test/test_emscripten.py
@@ -20,7 +20,7 @@
def check_node():
# cibuildwheel adds a pinned node version to the PATH
- # check it's in the PATH then, check it's the one that runs pyoodide
+ # check it's in the PATH then, check it's the one that runs pyodide
cibw_cache_path = Path(sys.argv[1]).resolve(strict=True)
# find the node executable in PATH
node = shutil.which("node")
From 0e3b3f1b8ae6d19cb3a2a4383da31b78d3e51228 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 11 Sep 2024 03:12:32 +0530
Subject: [PATCH 02/51] Add `pyodide_build_version` attribute
---
cibuildwheel/pyodide.py | 13 ++++++++++---
cibuildwheel/resources/build-platforms.toml | 2 +-
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 3b721a10a..eba3546e1 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -41,6 +41,7 @@ class PythonConfiguration:
version: str
identifier: str
pyodide_version: str
+ pyodide_build_version: str
emscripten_version: str
node_version: str
@@ -65,10 +66,14 @@ def install_emscripten(tmp: Path, version: str) -> Path:
return emcc_path
-def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str:
+def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str:
+ # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
+ # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
+ # specify the pyodide-build version, which will set up the xbuildenv for the requested
+ # Pyodide version.
pyodide_root = (
CIBW_CACHE_PATH
- / f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root"
+ / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
)
with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
if pyodide_root.exists():
@@ -171,7 +176,9 @@ def setup_python(
env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
log.step("Installing Pyodide xbuildenv...")
- env["PYODIDE_ROOT"] = install_xbuildenv(env, python_configuration.pyodide_version)
+ env["PYODIDE_ROOT"] = install_xbuildenv(
+ env, python_configuration.pyodide_build_version, python_configuration.pyodide_version
+ )
return env
diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml
index f96e6897a..1c967c9f5 100644
--- a/cibuildwheel/resources/build-platforms.toml
+++ b/cibuildwheel/resources/build-platforms.toml
@@ -172,5 +172,5 @@ python_configurations = [
[pyodide]
python_configurations = [
- { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", emscripten_version = "3.1.58", node_version = "v20" },
+ { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.28.0", emscripten_version = "3.1.58", node_version = "v20" },
]
From 394459fe359b26afab787f026dda7772b998c9ee Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 11 Sep 2024 03:28:26 +0530
Subject: [PATCH 03/51] Add version to xbuildenv log step
---
cibuildwheel/pyodide.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index eba3546e1..133ece0a0 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -175,7 +175,7 @@ def setup_python(
env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
- log.step("Installing Pyodide xbuildenv...")
+ log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...")
env["PYODIDE_ROOT"] = install_xbuildenv(
env, python_configuration.pyodide_build_version, python_configuration.pyodide_version
)
From dff7bf265ac0fb894c6cf35f4851daff3adc1c65 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 11 Sep 2024 03:39:41 +0530
Subject: [PATCH 04/51] Add version to Emscripten log step
---
cibuildwheel/pyodide.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 133ece0a0..4b8bef432 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -170,7 +170,7 @@ def setup_python(
env=env,
)
- log.step("Installing emscripten...")
+ log.step(f"Installing Emscripten version: {python_configuration.emscripten_version} ...")
emcc_path = install_emscripten(tmp, python_configuration.emscripten_version)
env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
From 16057bca6e211cd282f6981871252110d342f373 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 11 Sep 2024 03:39:04 +0530
Subject: [PATCH 05/51] Use `pyodide-build`'s version for updating constraints
---
noxfile.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/noxfile.py b/noxfile.py
index 0c2057b6d..4f59a98d8 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -102,10 +102,10 @@ def update_constraints(session: nox.Session) -> None:
pyodides = build_platforms["pyodide"]["python_configurations"]
for pyodide in pyodides:
python_version = ".".join(pyodide["version"].split(".")[:2])
- pyodide_version = pyodide["pyodide_version"]
+ pyodide_build_version = pyodide["pyodide_build_version"]
output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt"
tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in"
- tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_version}")
+ tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_build_version}")
session.run(
"uv",
"pip",
From f167c50cf570790a11dd4e608978321bb7f742cb Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 11 Sep 2024 03:38:23 +0530
Subject: [PATCH 06/51] Bump Pyodide constraints by updating `pyodide-build`
---
.../resources/constraints-pyodide312.txt | 32 +++++++------------
1 file changed, 11 insertions(+), 21 deletions(-)
diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt
index 890a0736e..57bcb2c89 100644
--- a/cibuildwheel/resources/constraints-pyodide312.txt
+++ b/cibuildwheel/resources/constraints-pyodide312.txt
@@ -6,7 +6,7 @@ anyio==4.4.0
# via httpx
auditwheel-emscripten==0.0.16
# via pyodide-build
-build==1.2.1
+build==1.2.2
# via
# -r .nox/update_constraints/tmp/constraints-pyodide.in
# pyodide-build
@@ -19,13 +19,11 @@ charset-normalizer==3.3.2
# via requests
click==8.1.7
# via typer
-cloudpickle==3.0.0
- # via loky
-cmake==3.30.2
+cmake==3.30.3
# via pyodide-build
distlib==0.3.8
# via virtualenv
-filelock==3.15.4
+filelock==3.16.0
# via virtualenv
h11==0.14.0
# via httpcore
@@ -40,8 +38,6 @@ idna==3.8
# requests
leb128==1.0.8
# via auditwheel-emscripten
-loky==3.4.1
- # via pyodide-build
markdown-it-py==3.0.0
# via rich
mdurl==0.1.2
@@ -54,33 +50,31 @@ packaging==24.1
# unearth
pip==24.2
# via -r .nox/update_constraints/tmp/constraints-pyodide.in
-platformdirs==4.2.2
+platformdirs==4.3.2
# via virtualenv
-pydantic==2.8.2
+pydantic==2.9.1
# via
# pyodide-build
# pyodide-lock
-pydantic-core==2.20.1
+pydantic-core==2.23.3
# via pydantic
pygments==2.18.0
# via rich
-pyodide-build==0.26.1
+pyodide-build==0.28.0
# via -r .nox/update_constraints/tmp/constraints-pyodide.in
pyodide-cli==0.2.4
# via
# auditwheel-emscripten
# pyodide-build
-pyodide-lock==0.1.0a6
+pyodide-lock==0.1.0a7
# via pyodide-build
pyproject-hooks==1.1.0
# via build
-pyyaml==6.0.2
- # via pyodide-build
requests==2.32.3
# via pyodide-build
resolvelib==1.0.1
# via pyodide-build
-rich==13.8.0
+rich==13.8.1
# via
# pyodide-build
# pyodide-cli
@@ -100,8 +94,6 @@ typer==0.12.5
# auditwheel-emscripten
# pyodide-build
# pyodide-cli
-types-requests==2.32.0.20240712
- # via pyodide-build
typing-extensions==4.12.2
# via
# pydantic
@@ -110,10 +102,8 @@ typing-extensions==4.12.2
unearth==0.17.2
# via pyodide-build
urllib3==2.2.2
- # via
- # requests
- # types-requests
-virtualenv==20.26.3
+ # via requests
+virtualenv==20.26.4
# via
# build
# pyodide-build
From 31a6be9557e91bac8c1ad7ac8090f139017c47dd Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 12 Sep 2024 18:49:36 +0530
Subject: [PATCH 07/51] Add a schema for `pyodide-version`
---
cibuildwheel/resources/cibuildwheel.schema.json | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json
index 976751a55..714c73067 100644
--- a/cibuildwheel/resources/cibuildwheel.schema.json
+++ b/cibuildwheel/resources/cibuildwheel.schema.json
@@ -814,6 +814,11 @@
},
"test-requires": {
"$ref": "#/properties/test-requires"
+ },
+ "pyodide-version": {
+ "description": "Specify the Pyodide xbuildenv version to use for building",
+ "type": "string",
+ "title": "CIBW_PYODIDE_VERSION"
}
}
}
From 90030679658b4e761f8a2f0c66332db7f9c72d85 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Fri, 13 Sep 2024 05:09:43 +0530
Subject: [PATCH 08/51] Update Pyodide constraints
---
.../resources/constraints-pyodide312.txt | 16 ++++------------
1 file changed, 4 insertions(+), 12 deletions(-)
diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt
index a86e06e9a..25a4835c0 100644
--- a/cibuildwheel/resources/constraints-pyodide312.txt
+++ b/cibuildwheel/resources/constraints-pyodide312.txt
@@ -19,8 +19,6 @@ charset-normalizer==3.3.2
# via requests
click==8.1.7
# via typer
-cloudpickle==3.0.0
- # via loky
cmake==3.30.3
# via pyodide-build
distlib==0.3.8
@@ -54,11 +52,11 @@ pip==24.2
# via -r .nox/update_constraints/tmp/constraints-pyodide.in
platformdirs==4.3.2
# via virtualenv
-pydantic==2.9.0
+pydantic==2.9.1
# via
# pyodide-build
# pyodide-lock
-pydantic-core==2.23.2
+pydantic-core==2.23.3
# via pydantic
pygments==2.18.0
# via rich
@@ -96,21 +94,15 @@ typer==0.12.5
# auditwheel-emscripten
# pyodide-build
# pyodide-cli
-types-requests==2.32.0.20240907
- # via pyodide-build
typing-extensions==4.12.2
# via
# pydantic
# pydantic-core
# typer
-tzdata==2024.1
- # via pydantic
unearth==0.17.2
# via pyodide-build
-urllib3==2.2.2
- # via
- # requests
- # types-requests
+urllib3==2.2.3
+ # via requests
virtualenv==20.26.4
# via
# build
From d8a8d5e8d73267042e8e0743779029516947fca6 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 21:40:47 +0530
Subject: [PATCH 09/51] Bump `pyodide-build` to new 0.29.0
---
.../resources/constraints-pyodide312.txt | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt
index 25a4835c0..2c80f2f9c 100644
--- a/cibuildwheel/resources/constraints-pyodide312.txt
+++ b/cibuildwheel/resources/constraints-pyodide312.txt
@@ -2,7 +2,7 @@
# nox -s update_constraints
annotated-types==0.7.0
# via pydantic
-anyio==4.4.0
+anyio==4.6.0
# via httpx
auditwheel-emscripten==0.0.16
# via pyodide-build
@@ -23,7 +23,7 @@ cmake==3.30.3
# via pyodide-build
distlib==0.3.8
# via virtualenv
-filelock==3.16.0
+filelock==3.16.1
# via virtualenv
h11==0.14.0
# via httpcore
@@ -31,7 +31,7 @@ httpcore==1.0.5
# via httpx
httpx==0.27.2
# via unearth
-idna==3.8
+idna==3.10
# via
# anyio
# httpx
@@ -50,17 +50,17 @@ packaging==24.1
# unearth
pip==24.2
# via -r .nox/update_constraints/tmp/constraints-pyodide.in
-platformdirs==4.3.2
+platformdirs==4.3.6
# via virtualenv
-pydantic==2.9.1
+pydantic==2.9.2
# via
# pyodide-build
# pyodide-lock
-pydantic-core==2.23.3
+pydantic-core==2.23.4
# via pydantic
pygments==2.18.0
# via rich
-pyodide-build==0.28.0
+pyodide-build==0.29.0
# via -r .nox/update_constraints/tmp/constraints-pyodide.in
pyodide-cli==0.2.4
# via
@@ -103,7 +103,7 @@ unearth==0.17.2
# via pyodide-build
urllib3==2.2.3
# via requests
-virtualenv==20.26.4
+virtualenv==20.26.5
# via
# build
# pyodide-build
From cf5dd3ec9ac65c6e7121ea6d8d4d58158df634bd Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 21:41:00 +0530
Subject: [PATCH 10/51] Test out another Pyodide identifier
---
cibuildwheel/resources/build-platforms.toml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml
index 03fccc9d9..30d5f5a3c 100644
--- a/cibuildwheel/resources/build-platforms.toml
+++ b/cibuildwheel/resources/build-platforms.toml
@@ -172,5 +172,6 @@ python_configurations = [
[pyodide]
python_configurations = [
- { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.28.0", emscripten_version = "3.1.58", node_version = "v20" },
+ { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
+ { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
]
From 4fe86e0bdc90aaa9bb4afc6d66f53f1b0daec814 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 21:47:34 +0530
Subject: [PATCH 11/51] Update outdated Pyodide constraints
---
cibuildwheel/resources/constraints-pyodide312.txt | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt
index ebb1c6305..2c80f2f9c 100644
--- a/cibuildwheel/resources/constraints-pyodide312.txt
+++ b/cibuildwheel/resources/constraints-pyodide312.txt
@@ -94,8 +94,6 @@ typer==0.12.5
# auditwheel-emscripten
# pyodide-build
# pyodide-cli
-types-requests==2.32.0.20240914
- # via pyodide-build
typing-extensions==4.12.2
# via
# pydantic
@@ -104,9 +102,7 @@ typing-extensions==4.12.2
unearth==0.17.2
# via pyodide-build
urllib3==2.2.3
- # via
- # requests
- # types-requests
+ # via requests
virtualenv==20.26.5
# via
# build
From b6830ee4dad18274d60e071655142b70444c971d Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 22:25:47 +0530
Subject: [PATCH 12/51] Add Pyodide version to temp directory name
---
cibuildwheel/pyodide.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 4b8bef432..7253202a5 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -226,7 +226,8 @@ def build(options: Options, tmp_path: Path) -> None:
log.build_start(config.identifier)
- identifier_tmp_dir = tmp_path / config.identifier
+ # Include both the identifier and the Pyodide version in the temp directory name
+ identifier_tmp_dir = tmp_path / f"{config.identifier}_{config.pyodide_version}"
built_wheel_dir = identifier_tmp_dir / "built_wheel"
repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
identifier_tmp_dir.mkdir()
From aaf32e594d44c1259bc227cbd5e14ac014e8d8c5 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 22:34:58 +0530
Subject: [PATCH 13/51] Remove Pyodide 0.26.1 from build configurations
---
cibuildwheel/resources/build-platforms.toml | 1 -
1 file changed, 1 deletion(-)
diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml
index 30d5f5a3c..15a711f54 100644
--- a/cibuildwheel/resources/build-platforms.toml
+++ b/cibuildwheel/resources/build-platforms.toml
@@ -172,6 +172,5 @@ python_configurations = [
[pyodide]
python_configurations = [
- { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
{ identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
]
From bb6e0d67f85f464ab2ffcf0c3c743b3855b67b4d Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 22:57:25 +0530
Subject: [PATCH 14/51] Retrieve + validate + install specific xbuildenvs
---
cibuildwheel/pyodide.py | 41 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 41 insertions(+)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 7253202a5..50e0be661 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import json
import os
import shutil
import sys
@@ -66,6 +67,36 @@ def install_emscripten(tmp: Path, version: str) -> Path:
return emcc_path
+def retrieve_compatible_xbuildenvs() -> list[str]:
+ """Search for compatible xbuildenvs for the current pyodide-build version."""
+ xbuildenvs = call(
+ "pyodide",
+ "xbuildenv",
+ "search",
+ "--json",
+ "--all",
+ capture_stdout=True,
+ ).strip()
+
+ xbuildenvs_dict = json.loads(xbuildenvs)
+ compatible_xbuildenvs = [
+ env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
+ ]
+
+ return compatible_xbuildenvs
+
+
+def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None:
+ """Validate that the requested xbuildenv version is compatible with the pyodide-build version."""
+ xbuildenvs = retrieve_compatible_xbuildenvs()
+ if xbuildenv_version not in xbuildenvs:
+ msg = (
+ f"The xbuildenv version {xbuildenv_version} is not compatible with the pyodide-build version {pyodide_build_version}."
+ f" The compatible versions are: {xbuildenvs}. Please use the 'pyodide xbuildenv search' command to find the compatible versions."
+ )
+ raise errors.FatalError(msg)
+
+
def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str:
# Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
# not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
@@ -75,6 +106,16 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v
CIBW_CACHE_PATH
/ f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
)
+
+ # Validate that the requested xbuildenv version is compatible with the pyodide-build version.
+ # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
+ # which will always be compatible. Hence, this function really only checks for the case where the
+ # version is supplied manually through CIBW_PYODIDE_VERSION environment variable.
+ cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION")
+ if cibw_pyodide_version:
+ validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version)
+ pyodide_version = cibw_pyodide_version
+
with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
if pyodide_root.exists():
return str(pyodide_root)
From 735d5bb100705a6225ae90e959a2b031750ec55b Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 22:58:38 +0530
Subject: [PATCH 15/51] Test wheel builds with Pyodide 0.26.2
---
.github/workflows/test.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 54e950e80..68a92cabf 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -194,6 +194,7 @@ jobs:
output-dir: wheelhouse
env:
CIBW_PLATFORM: pyodide
+ CIBW_PYODIDE_VERSION: 0.26.2
- name: Run tests with 'CIBW_PLATFORM' set to 'pyodide'
run: |
From b8ac6c015fff7cf3312d57683874eda238dd3f5d Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 23:09:51 +0530
Subject: [PATCH 16/51] Add correct Pyodide version to identifier temp dir
---
cibuildwheel/pyodide.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 50e0be661..f019c0202 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -268,7 +268,9 @@ def build(options: Options, tmp_path: Path) -> None:
log.build_start(config.identifier)
# Include both the identifier and the Pyodide version in the temp directory name
- identifier_tmp_dir = tmp_path / f"{config.identifier}_{config.pyodide_version}"
+ cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", config.pyodide_version)
+ identifier_tmp_dir = tmp_path / f"{config.identifier}_{cibw_pyodide_version}"
+
built_wheel_dir = identifier_tmp_dir / "built_wheel"
repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
identifier_tmp_dir.mkdir()
From 7796311dc62ec42461534404cb9dbffc5b77a05a Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 23:19:59 +0530
Subject: [PATCH 17/51] Don't pre-call Pyodide xbuildenv search
---
cibuildwheel/pyodide.py | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index f019c0202..3d3778305 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -107,19 +107,19 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v
/ f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
)
- # Validate that the requested xbuildenv version is compatible with the pyodide-build version.
- # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
- # which will always be compatible. Hence, this function really only checks for the case where the
- # version is supplied manually through CIBW_PYODIDE_VERSION environment variable.
- cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION")
- if cibw_pyodide_version:
- validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version)
- pyodide_version = cibw_pyodide_version
-
with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
if pyodide_root.exists():
return str(pyodide_root)
+ # Validate that the requested xbuildenv version is compatible with the pyodide-build version.
+ # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
+ # which will always be compatible. Hence, this function really only checks for the case where the
+ # version is supplied manually through CIBW_PYODIDE_VERSION environment variable.
+ cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION")
+ if cibw_pyodide_version:
+ validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version)
+ pyodide_version = cibw_pyodide_version
+
# We don't want to mutate env but we need to delete any existing
# PYODIDE_ROOT so copy it first.
env = dict(env)
From 97e22c882534dec8ee686e51b2813735e16d9ee9 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 24 Sep 2024 23:20:31 +0530
Subject: [PATCH 18/51] Fetch just the stable Pyodide versions
---
cibuildwheel/pyodide.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 3d3778305..68b3a529e 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -83,7 +83,12 @@ def retrieve_compatible_xbuildenvs() -> list[str]:
env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
]
- return compatible_xbuildenvs
+ # Fetch just the "stable" versions
+ compatible_xbuildenvs_filtered = [
+ version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
+ ]
+
+ return compatible_xbuildenvs_filtered
def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None:
From d30eb6aa882ff0ea3d3061fb8f2193bf4b5f6097 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 00:04:09 +0530
Subject: [PATCH 19/51] Refactor search + validation + install into one step
---
cibuildwheel/pyodide.py | 61 ++++++++++++++++++-----------------------
1 file changed, 26 insertions(+), 35 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 68b3a529e..4c9f24ba2 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -67,8 +67,20 @@ def install_emscripten(tmp: Path, version: str) -> Path:
return emcc_path
-def retrieve_compatible_xbuildenvs() -> list[str]:
- """Search for compatible xbuildenvs for the current pyodide-build version."""
+def search_and_install_xbuildenv(
+ env: dict[str, str], pyodide_build_version: str, pyodide_version: str
+) -> str:
+ """Searches, validates, and installs a particular Pyodide xbuildenv version."""
+ # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
+ # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
+ # specify the pyodide-build version in the root path, which will set up the xbuildenv for
+ # the requested Pyodide version.
+ pyodide_root = (
+ CIBW_CACHE_PATH
+ / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
+ )
+
+ # Search for compatible xbuildenvs for the current pyodide-build version
xbuildenvs = call(
"pyodide",
"xbuildenv",
@@ -77,54 +89,33 @@ def retrieve_compatible_xbuildenvs() -> list[str]:
"--all",
capture_stdout=True,
).strip()
-
xbuildenvs_dict = json.loads(xbuildenvs)
compatible_xbuildenvs = [
env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
]
-
# Fetch just the "stable" versions
compatible_xbuildenvs_filtered = [
version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
]
- return compatible_xbuildenvs_filtered
-
-
-def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None:
- """Validate that the requested xbuildenv version is compatible with the pyodide-build version."""
- xbuildenvs = retrieve_compatible_xbuildenvs()
- if xbuildenv_version not in xbuildenvs:
+ # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version.
+ # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
+ # which will always be compatible. Hence, this condition really only checks for the case where the
+ # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable.
+ cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version)
+ if cibw_pyodide_version not in compatible_xbuildenvs_filtered:
msg = (
- f"The xbuildenv version {xbuildenv_version} is not compatible with the pyodide-build version {pyodide_build_version}."
- f" The compatible versions are: {xbuildenvs}. Please use the 'pyodide xbuildenv search' command to find the compatible versions."
+ f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
+ f" version {pyodide_build_version}. The compatible versions available to download are:"
+ f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to"
+ f" find the compatible versions for {pyodide_build_version}"
)
raise errors.FatalError(msg)
-
-def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str:
- # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
- # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
- # specify the pyodide-build version, which will set up the xbuildenv for the requested
- # Pyodide version.
- pyodide_root = (
- CIBW_CACHE_PATH
- / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
- )
-
with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
if pyodide_root.exists():
return str(pyodide_root)
- # Validate that the requested xbuildenv version is compatible with the pyodide-build version.
- # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
- # which will always be compatible. Hence, this function really only checks for the case where the
- # version is supplied manually through CIBW_PYODIDE_VERSION environment variable.
- cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION")
- if cibw_pyodide_version:
- validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version)
- pyodide_version = cibw_pyodide_version
-
# We don't want to mutate env but we need to delete any existing
# PYODIDE_ROOT so copy it first.
env = dict(env)
@@ -133,7 +124,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v
"pyodide",
"xbuildenv",
"install",
- pyodide_version,
+ cibw_pyodide_version,
env=env,
cwd=CIBW_CACHE_PATH,
)
@@ -222,7 +213,7 @@ def setup_python(
env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...")
- env["PYODIDE_ROOT"] = install_xbuildenv(
+ env["PYODIDE_ROOT"] = search_and_install_xbuildenv(
env, python_configuration.pyodide_build_version, python_configuration.pyodide_version
)
From ce2a3f08b9725c7ddebea67f86a6efbe72124f6d Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 00:20:46 +0530
Subject: [PATCH 20/51] Move all of it under a lock
---
cibuildwheel/pyodide.py | 64 ++++++++++++++++++++---------------------
1 file changed, 32 insertions(+), 32 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 4c9f24ba2..ff46e5fd4 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -80,42 +80,42 @@ def search_and_install_xbuildenv(
/ f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
)
- # Search for compatible xbuildenvs for the current pyodide-build version
- xbuildenvs = call(
- "pyodide",
- "xbuildenv",
- "search",
- "--json",
- "--all",
- capture_stdout=True,
- ).strip()
- xbuildenvs_dict = json.loads(xbuildenvs)
- compatible_xbuildenvs = [
- env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
- ]
- # Fetch just the "stable" versions
- compatible_xbuildenvs_filtered = [
- version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
- ]
-
- # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version.
- # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
- # which will always be compatible. Hence, this condition really only checks for the case where the
- # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable.
- cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version)
- if cibw_pyodide_version not in compatible_xbuildenvs_filtered:
- msg = (
- f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
- f" version {pyodide_build_version}. The compatible versions available to download are:"
- f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to"
- f" find the compatible versions for {pyodide_build_version}"
- )
- raise errors.FatalError(msg)
-
with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
if pyodide_root.exists():
return str(pyodide_root)
+ # Search for compatible xbuildenvs for the current pyodide-build version
+ xbuildenvs = call(
+ "pyodide",
+ "xbuildenv",
+ "search",
+ "--json",
+ "--all",
+ capture_stdout=True,
+ ).strip()
+ xbuildenvs_dict = json.loads(xbuildenvs)
+ compatible_xbuildenvs = [
+ env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
+ ]
+ # Fetch just the "stable" versions
+ compatible_xbuildenvs_filtered = [
+ version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
+ ]
+
+ # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version.
+ # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
+ # which will always be compatible. Hence, this condition really only checks for the case where the
+ # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable.
+ cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version)
+ if cibw_pyodide_version not in compatible_xbuildenvs_filtered:
+ msg = (
+ f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
+ f" version {pyodide_build_version}. The compatible versions available to download are:"
+ f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to"
+ f" find the compatible versions for {pyodide_build_version}"
+ )
+ raise errors.FatalError(msg)
+
# We don't want to mutate env but we need to delete any existing
# PYODIDE_ROOT so copy it first.
env = dict(env)
From 13fbf66873c7e1a270d03da2b1824ff70b4f0eb4 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 00:28:48 +0530
Subject: [PATCH 21/51] Reorder xbuildenv installation
---
cibuildwheel/pyodide.py | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index ff46e5fd4..353d9b71b 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -84,7 +84,12 @@ def search_and_install_xbuildenv(
if pyodide_root.exists():
return str(pyodide_root)
- # Search for compatible xbuildenvs for the current pyodide-build version
+ # We don't want to mutate env but we need to delete any existing
+ # PYODIDE_ROOT so copy it first.
+ env = dict(env)
+ env.pop("PYODIDE_ROOT", None)
+
+ # 1. Search for compatible xbuildenvs for the current pyodide-build version
xbuildenvs = call(
"pyodide",
"xbuildenv",
@@ -102,7 +107,7 @@ def search_and_install_xbuildenv(
version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
]
- # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version.
+ # 2. Now, validate that the requested xbuildenv version is compatible with the pyodide-build version.
# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
# which will always be compatible. Hence, this condition really only checks for the case where the
# version is supplied manually through the CIBW_PYODIDE_VERSION environment variable.
@@ -116,10 +121,7 @@ def search_and_install_xbuildenv(
)
raise errors.FatalError(msg)
- # We don't want to mutate env but we need to delete any existing
- # PYODIDE_ROOT so copy it first.
- env = dict(env)
- env.pop("PYODIDE_ROOT", None)
+ # 3. Install the xbuildenv
call(
"pyodide",
"xbuildenv",
From 14ec0712667dcca54e18efb83012cb7bd3ee0564 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 00:34:49 +0530
Subject: [PATCH 22/51] Add env and cwd to xbuildenv search call
---
cibuildwheel/pyodide.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 353d9b71b..04dcfe732 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -96,6 +96,8 @@ def search_and_install_xbuildenv(
"search",
"--json",
"--all",
+ env=env,
+ cwd=CIBW_CACHE_PATH,
capture_stdout=True,
).strip()
xbuildenvs_dict = json.loads(xbuildenvs)
From aae64bbc1e2631156ee03cf21fd259f09c54e63f Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 00:39:33 +0530
Subject: [PATCH 23/51] Temporarily lower to 0.26.2 target
---
cibuildwheel/resources/build-platforms.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml
index 15a711f54..f2ca53ce5 100644
--- a/cibuildwheel/resources/build-platforms.toml
+++ b/cibuildwheel/resources/build-platforms.toml
@@ -172,5 +172,5 @@ python_configurations = [
[pyodide]
python_configurations = [
- { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
+ { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
]
From 6956121375229028d0d10b74c2689153097eba89 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 01:34:00 +0530
Subject: [PATCH 24/51] Separate out search, validate, install; again
---
cibuildwheel/pyodide.py | 97 +++++++++++++++++++++++------------------
1 file changed, 55 insertions(+), 42 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 04dcfe732..862df5603 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -67,10 +67,49 @@ def install_emscripten(tmp: Path, version: str) -> Path:
return emcc_path
-def search_and_install_xbuildenv(
- env: dict[str, str], pyodide_build_version: str, pyodide_version: str
-) -> str:
- """Searches, validates, and installs a particular Pyodide xbuildenv version."""
+def search_xbuildenv() -> list[str]:
+ """Searches for the compatible xbuildenvs for the current pyodide-build version"""
+ xbuildenvs = call(
+ "pyodide",
+ "xbuildenv",
+ "search",
+ "--json",
+ "--all",
+ capture_stdout=True,
+ ).strip()
+ xbuildenvs_dict = json.loads(xbuildenvs)
+ compatible_xbuildenvs = [
+ env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
+ ]
+ # Fetch just the "stable" versions
+ compatible_xbuildenvs_filtered = [
+ version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
+ ]
+
+ return compatible_xbuildenvs_filtered
+
+
+def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None:
+ """Validate the Pyodide version if set manually for the current pyodide-build version"""
+ pyodide_versions = search_xbuildenv()
+
+ # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
+ # which will always be compatible. Hence, this condition really checks only for the case where the
+ # version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an
+ # error as appropriate.
+ cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version)
+ if cibw_pyodide_version not in pyodide_versions:
+ msg = (
+ f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
+ f" version {pyodide_build_version}. The compatible versions available to download are:"
+ f" {pyodide_versions}. Please use the 'pyodide xbuildenv search' command to"
+ f" find the compatible versions for {pyodide_build_version}"
+ )
+ raise errors.FatalError(msg)
+
+
+def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str:
+ """Installs a particular Pyodide xbuildenv version."""
# Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
# not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
# specify the pyodide-build version in the root path, which will set up the xbuildenv for
@@ -89,46 +128,12 @@ def search_and_install_xbuildenv(
env = dict(env)
env.pop("PYODIDE_ROOT", None)
- # 1. Search for compatible xbuildenvs for the current pyodide-build version
- xbuildenvs = call(
- "pyodide",
- "xbuildenv",
- "search",
- "--json",
- "--all",
- env=env,
- cwd=CIBW_CACHE_PATH,
- capture_stdout=True,
- ).strip()
- xbuildenvs_dict = json.loads(xbuildenvs)
- compatible_xbuildenvs = [
- env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
- ]
- # Fetch just the "stable" versions
- compatible_xbuildenvs_filtered = [
- version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
- ]
-
- # 2. Now, validate that the requested xbuildenv version is compatible with the pyodide-build version.
- # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
- # which will always be compatible. Hence, this condition really only checks for the case where the
- # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable.
- cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version)
- if cibw_pyodide_version not in compatible_xbuildenvs_filtered:
- msg = (
- f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
- f" version {pyodide_build_version}. The compatible versions available to download are:"
- f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to"
- f" find the compatible versions for {pyodide_build_version}"
- )
- raise errors.FatalError(msg)
-
# 3. Install the xbuildenv
call(
"pyodide",
"xbuildenv",
"install",
- cibw_pyodide_version,
+ pyodide_version,
env=env,
cwd=CIBW_CACHE_PATH,
)
@@ -216,9 +221,17 @@ def setup_python(
env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
- log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...")
- env["PYODIDE_ROOT"] = search_and_install_xbuildenv(
- env, python_configuration.pyodide_build_version, python_configuration.pyodide_version
+ # Allow overriding the xbuildenv version with an environment variable. This allows
+ # testing new Pyodide xbuildenv versions before they are officially released, or for
+ # using a different Pyodide version other than the one that is listed to be compatible
+ # with the pyodide-build version in the build-platforms.toml file.
+ cibw_pyodide_version = os.environ.get(
+ "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version
+ )
+ log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...")
+ validate_xbuildenv(cibw_pyodide_version, cibw_pyodide_version)
+ env["PYODIDE_ROOT"] = install_xbuildenv(
+ env, python_configuration.pyodide_build_version, cibw_pyodide_version
)
return env
From e5d8443586839839109c3972c5ee4bb05462b332 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 01:38:46 +0530
Subject: [PATCH 25/51] Run xbuildenv search in `CIBW_CACHE_PATH`
---
cibuildwheel/pyodide.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 862df5603..590856176 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -75,6 +75,7 @@ def search_xbuildenv() -> list[str]:
"search",
"--json",
"--all",
+ cwd=CIBW_CACHE_PATH,
capture_stdout=True,
).strip()
xbuildenvs_dict = json.loads(xbuildenvs)
From 95c368116aad3b32a87b33169dda7251e6f1a396 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 01:54:57 +0530
Subject: [PATCH 26/51] Remove prior `PYODIDE_ROOT` env vars, copy envs
---
cibuildwheel/pyodide.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 590856176..079f2e0d8 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -69,12 +69,17 @@ def install_emscripten(tmp: Path, version: str) -> Path:
def search_xbuildenv() -> list[str]:
"""Searches for the compatible xbuildenvs for the current pyodide-build version"""
+
+ env = dict(os.environ)
+ env.pop("PYODIDE_ROOT", None)
+
xbuildenvs = call(
"pyodide",
"xbuildenv",
"search",
"--json",
"--all",
+ env=env,
cwd=CIBW_CACHE_PATH,
capture_stdout=True,
).strip()
@@ -86,6 +91,7 @@ def search_xbuildenv() -> list[str]:
compatible_xbuildenvs_filtered = [
version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
]
+ # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions
return compatible_xbuildenvs_filtered
@@ -110,7 +116,7 @@ def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None
def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str:
- """Installs a particular Pyodide xbuildenv version."""
+ """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root."""
# Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
# not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
# specify the pyodide-build version in the root path, which will set up the xbuildenv for
From 09af46f14c9dede60160727878f74f7fb292c029 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 02:08:03 +0530
Subject: [PATCH 27/51] Validate doesn't need to depend on searching
---
cibuildwheel/pyodide.py | 34 ++++++++++++++++++----------------
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 079f2e0d8..b0ac81dcb 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -67,12 +67,8 @@ def install_emscripten(tmp: Path, version: str) -> Path:
return emcc_path
-def search_xbuildenv() -> list[str]:
+def search_xbuildenv(env: dict[str, str]) -> list[str]:
"""Searches for the compatible xbuildenvs for the current pyodide-build version"""
-
- env = dict(os.environ)
- env.pop("PYODIDE_ROOT", None)
-
xbuildenvs = call(
"pyodide",
"xbuildenv",
@@ -96,20 +92,21 @@ def search_xbuildenv() -> list[str]:
return compatible_xbuildenvs_filtered
-def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None:
- """Validate the Pyodide version if set manually for the current pyodide-build version"""
- pyodide_versions = search_xbuildenv()
+# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
+# which will always be compatible. Hence, this condition really checks only for the case where the
+# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an
+# error as appropriate.
+def validate_xbuildenv(
+ cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str]
+) -> None:
+ """Validate the Pyodide version if set manually for the current pyodide-build version
+ against a list of compatible versions."""
- # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
- # which will always be compatible. Hence, this condition really checks only for the case where the
- # version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an
- # error as appropriate.
- cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version)
- if cibw_pyodide_version not in pyodide_versions:
+ if cibw_pyodide_version not in compatible_versions:
msg = (
f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
f" version {pyodide_build_version}. The compatible versions available to download are:"
- f" {pyodide_versions}. Please use the 'pyodide xbuildenv search' command to"
+ f" {compatible_versions}. Please use the 'pyodide xbuildenv search' command to"
f" find the compatible versions for {pyodide_build_version}"
)
raise errors.FatalError(msg)
@@ -236,7 +233,12 @@ def setup_python(
"CIBW_PYODIDE_VERSION", python_configuration.pyodide_version
)
log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...")
- validate_xbuildenv(cibw_pyodide_version, cibw_pyodide_version)
+ # Search for compatible xbuildenv versions
+ compatible_versions = search_xbuildenv(env)
+ # and then validate the xbuildenv version
+ validate_xbuildenv(
+ cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions
+ )
env["PYODIDE_ROOT"] = install_xbuildenv(
env, python_configuration.pyodide_build_version, cibw_pyodide_version
)
From 66999cbdc45e6734ff0d69fe3f8593cac4f69b47 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 02:08:47 +0530
Subject: [PATCH 28/51] Add file lock when searching xbuildenvs
---
cibuildwheel/pyodide.py | 43 +++++++++++++++++++++--------------------
1 file changed, 22 insertions(+), 21 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index b0ac81dcb..bc463da04 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -69,27 +69,28 @@ def install_emscripten(tmp: Path, version: str) -> Path:
def search_xbuildenv(env: dict[str, str]) -> list[str]:
"""Searches for the compatible xbuildenvs for the current pyodide-build version"""
- xbuildenvs = call(
- "pyodide",
- "xbuildenv",
- "search",
- "--json",
- "--all",
- env=env,
- cwd=CIBW_CACHE_PATH,
- capture_stdout=True,
- ).strip()
- xbuildenvs_dict = json.loads(xbuildenvs)
- compatible_xbuildenvs = [
- env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
- ]
- # Fetch just the "stable" versions
- compatible_xbuildenvs_filtered = [
- version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
- ]
- # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions
-
- return compatible_xbuildenvs_filtered
+ with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
+ xbuildenvs = call(
+ "pyodide",
+ "xbuildenv",
+ "search",
+ "--json",
+ "--all",
+ env=env,
+ cwd=CIBW_CACHE_PATH,
+ capture_stdout=True,
+ ).strip()
+ xbuildenvs_dict = json.loads(xbuildenvs)
+ compatible_xbuildenvs = [
+ env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
+ ]
+ # Fetch just the "stable" versions
+ compatible_xbuildenvs_filtered = [
+ version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
+ ]
+ # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions
+
+ return compatible_xbuildenvs_filtered
# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
From 1af1e5dd8464bd27dfff8134b4d73cb5b9a0b704 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Wed, 25 Sep 2024 03:06:25 +0530
Subject: [PATCH 29/51] Test the original version: 0.26.1
---
.github/workflows/test.yml | 1 -
cibuildwheel/resources/build-platforms.toml | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 68a92cabf..54e950e80 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -194,7 +194,6 @@ jobs:
output-dir: wheelhouse
env:
CIBW_PLATFORM: pyodide
- CIBW_PYODIDE_VERSION: 0.26.2
- name: Run tests with 'CIBW_PLATFORM' set to 'pyodide'
run: |
diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml
index f2ca53ce5..9c4cc2a4c 100644
--- a/cibuildwheel/resources/build-platforms.toml
+++ b/cibuildwheel/resources/build-platforms.toml
@@ -172,5 +172,5 @@ python_configurations = [
[pyodide]
python_configurations = [
- { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
+ { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
]
From d3445483bc5ffde485485afb8f0980462084cb7a Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Tue, 22 Oct 2024 17:22:32 +0530
Subject: [PATCH 30/51] Update Pyodide constraints
---
cibuildwheel/resources/constraints-pyodide312.txt | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt
index 394806f20..f575ad707 100644
--- a/cibuildwheel/resources/constraints-pyodide312.txt
+++ b/cibuildwheel/resources/constraints-pyodide312.txt
@@ -19,8 +19,6 @@ charset-normalizer==3.4.0
# via requests
click==8.1.7
# via typer
-cloudpickle==3.1.0
- # via loky
cmake==3.30.5
# via pyodide-build
distlib==0.3.9
@@ -96,8 +94,6 @@ typer==0.12.5
# auditwheel-emscripten
# pyodide-build
# pyodide-cli
-types-requests==2.32.0.20241016
- # via pyodide-build
typing-extensions==4.12.2
# via
# pydantic
@@ -106,9 +102,7 @@ typing-extensions==4.12.2
unearth==0.17.2
# via pyodide-build
urllib3==2.2.3
- # via
- # requests
- # types-requests
+ # via requests
virtualenv==20.27.0
# via
# build
From 738ed1f5e5a18b7d5a67c19d43820eeef789dd1d Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 14:03:58 +0530
Subject: [PATCH 31/51] Update constraints for `pyodide-build` 0.29.0 again
---
cibuildwheel/resources/constraints-pyodide312.txt | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt
index 1920c59da..a0fcc3bee 100644
--- a/cibuildwheel/resources/constraints-pyodide312.txt
+++ b/cibuildwheel/resources/constraints-pyodide312.txt
@@ -19,8 +19,6 @@ charset-normalizer==3.4.0
# via requests
click==8.1.7
# via typer
-cloudpickle==3.1.0
- # via loky
cmake==3.31.0.1
# via pyodide-build
distlib==0.3.9
@@ -54,11 +52,11 @@ pip==24.3.1
# via -r .nox/update_constraints/tmp/constraints-pyodide.in
platformdirs==4.3.6
# via virtualenv
-pydantic==2.9.2
+pydantic==2.10.0
# via
# pyodide-build
# pyodide-lock
-pydantic-core==2.23.4
+pydantic-core==2.27.0
# via pydantic
pygments==2.18.0
# via rich
@@ -91,7 +89,7 @@ sniffio==1.3.1
# via
# anyio
# httpx
-typer==0.13.0
+typer==0.13.1
# via
# auditwheel-emscripten
# pyodide-build
@@ -104,9 +102,7 @@ typing-extensions==4.12.2
unearth==0.17.2
# via pyodide-build
urllib3==2.2.3
- # via
- # requests
- # types-requests
+ # via requests
virtualenv==20.27.1
# via
# build
From 41e466a487b3d411b6dbd4847ad82fab404fb94e Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 14:12:22 +0530
Subject: [PATCH 32/51] =?UTF-8?q?Bump=20Pyodide=20from=20version=200.26.1?=
=?UTF-8?q?=20=E2=9E=A1=EF=B8=8F=20version=200.26.4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
cibuildwheel/resources/build-platforms.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml
index 8cfa436e6..6e2d296b2 100644
--- a/cibuildwheel/resources/build-platforms.toml
+++ b/cibuildwheel/resources/build-platforms.toml
@@ -190,5 +190,5 @@ python_configurations = [
[pyodide]
python_configurations = [
- { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
+ { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.4", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
]
From 2a66ddd0a8e6ed531c5e033a2992ad8435b627ee Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:15:44 +0530
Subject: [PATCH 33/51] Add note on compatibility for macOS + other archs
---
docs/options.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/options.md b/docs/options.md
index 327bfe1f8..3970c3cb2 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -255,7 +255,7 @@ Default: `auto`
- For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows.
- For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows.
-- For `pyodide` you need to be on an x86-64 linux runner and `python3.12` must be available in `PATH`.
+- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may work, too. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information.
This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config.
From dff72caa48423f85c50d30483c2e13421a6436e9 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:15:58 +0530
Subject: [PATCH 34/51] Note Pyodide version for Pyodide identifier
---
docs/options.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/options.md b/docs/options.md
index 3970c3cb2..4ce75774c 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -321,7 +321,7 @@ If you set the value lower, cibuildwheel will cap it to the lowest supported val
Windows arm64 platform support is experimental.
For an experimental WebAssembly build with `--platform pyodide`,
-`cp312-pyodide_wasm32` is the only platform identifier.
+`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.26.4`](https://github.com/pyodide/pyodide/releases/tag/0.26.4).
See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7.
From 9a78845d3d7d8852f495a6db66f369ac54f124d4 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:16:14 +0530
Subject: [PATCH 35/51] Docs about `CIBW_PYODIDE_VERSION`
---
docs/setup.md | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/docs/setup.md b/docs/setup.md
index 32e0d01c5..e900eef4b 100644
--- a/docs/setup.md
+++ b/docs/setup.md
@@ -113,7 +113,14 @@ Pre-requisite: you need to have a matching host version of Python (unlike all
other cibuildwheel platforms). Linux host highly recommended; macOS hosts may
work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work.
-You must target pyodide with `--platform pyodide` (or use `--only` on the identifier).
+You must target Pyodide with `--platform pyodide` (or use `--only` on the identifier).
+
+!!!tip
+
+ It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` environment variable to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json) or can be
+ referenced using the `pyodide xbuildenv search` command.
+
+ This option is **not available** in the `pyproject.toml` config.
## Configure a CI service
From e551021a1bf71d8966baa6ace43adf162a6f7e23 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 15:47:09 +0530
Subject: [PATCH 36/51] Don't fetch just the stable versions
---
cibuildwheel/pyodide.py | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 6c547124d..843e5339f 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -81,16 +81,11 @@ def search_xbuildenv(env: dict[str, str]) -> list[str]:
capture_stdout=True,
).strip()
xbuildenvs_dict = json.loads(xbuildenvs)
- compatible_xbuildenvs = [
+ compatible_xbuildenv_versions = [
env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
]
- # Fetch just the "stable" versions
- compatible_xbuildenvs_filtered = [
- version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
- ]
- # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions
- return compatible_xbuildenvs_filtered
+ return compatible_xbuildenv_versions
# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
From 1fe8a04d0a7c72b55697685dd358d47756aac54d Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 18:26:16 +0530
Subject: [PATCH 37/51] Discard a variable that's not used later
---
cibuildwheel/pyodide.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 843e5339f..44c5ccbb9 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -82,7 +82,7 @@ def search_xbuildenv(env: dict[str, str]) -> list[str]:
).strip()
xbuildenvs_dict = json.loads(xbuildenvs)
compatible_xbuildenv_versions = [
- env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
+ _["version"] for _ in xbuildenvs_dict["environments"] if _["compatible"]
]
return compatible_xbuildenv_versions
From 8a8177c646f365bb8654f9f5560446acadf246bd Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 18:29:02 +0530
Subject: [PATCH 38/51] =?UTF-8?q?Rename=20`search=5Fxbuildenv`=20=E2=9E=A1?=
=?UTF-8?q?=EF=B8=8F=20`get=5Fxbuildenv=5Fversions`?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
cibuildwheel/pyodide.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 44c5ccbb9..b6cb1a427 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -67,7 +67,7 @@ def install_emscripten(tmp: Path, version: str) -> Path:
return emcc_path
-def search_xbuildenv(env: dict[str, str]) -> list[str]:
+def get_xbuildenv_versions(env: dict[str, str]) -> list[str]:
"""Searches for the compatible xbuildenvs for the current pyodide-build version"""
with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
xbuildenvs = call(
@@ -230,7 +230,7 @@ def setup_python(
)
log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...")
# Search for compatible xbuildenv versions
- compatible_versions = search_xbuildenv(env)
+ compatible_versions = get_xbuildenv_versions(env)
# and then validate the xbuildenv version
validate_xbuildenv(
cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions
From fffb7050aa3eb15147b9409f2ca30f6f01092a8a Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 18:29:32 +0530
Subject: [PATCH 39/51] =?UTF-8?q?`validate=5Fxbuildenv`=20=E2=9E=A1?=
=?UTF-8?q?=EF=B8=8F=20`validate=5Fxbuildenv=5Fversion`?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
cibuildwheel/pyodide.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index b6cb1a427..2f403d499 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -92,7 +92,7 @@ def get_xbuildenv_versions(env: dict[str, str]) -> list[str]:
# which will always be compatible. Hence, this condition really checks only for the case where the
# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an
# error as appropriate.
-def validate_xbuildenv(
+def validate_xbuildenv_version(
cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str]
) -> None:
"""Validate the Pyodide version if set manually for the current pyodide-build version
@@ -232,7 +232,7 @@ def setup_python(
# Search for compatible xbuildenv versions
compatible_versions = get_xbuildenv_versions(env)
# and then validate the xbuildenv version
- validate_xbuildenv(
+ validate_xbuildenv_version(
cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions
)
env["PYODIDE_ROOT"] = install_xbuildenv(
From 0df3c45f28761a16f56bee9441805f8e8f2ab23b Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 18:36:58 +0530
Subject: [PATCH 40/51] Replace ordered comment, add newline
---
cibuildwheel/pyodide.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 2f403d499..6fb4447ae 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -101,9 +101,9 @@ def validate_xbuildenv_version(
if cibw_pyodide_version not in compatible_versions:
msg = (
f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
- f" version {pyodide_build_version}. The compatible versions available to download are:"
- f" {compatible_versions}. Please use the 'pyodide xbuildenv search' command to"
- f" find the compatible versions for {pyodide_build_version}"
+ f" version {pyodide_build_version}. The compatible versions available to download are:\n"
+ f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to"
+ f" find the compatible versions for {pyodide_build_version}."
)
raise errors.FatalError(msg)
@@ -128,7 +128,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v
env = dict(env)
env.pop("PYODIDE_ROOT", None)
- # 3. Install the xbuildenv
+ # Install the xbuildenv
call(
"pyodide",
"xbuildenv",
From 057e542fa3a670dd4e92773e0e33a12b806dbc6a Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 18:37:53 +0530
Subject: [PATCH 41/51] Replace sentence on macOS support
Co-Authored-By: Hood Chatham <roberthoodchatham@gmail.com>
---
docs/options.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/options.md b/docs/options.md
index 4ce75774c..8281836ef 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -255,7 +255,7 @@ Default: `auto`
- For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows.
- For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows.
-- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may work, too. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information.
+- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts work, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information.
This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config.
From 29b5effd8c586d5a2705c43304ba4c4922dbc87c Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 18:40:31 +0530
Subject: [PATCH 42/51] =?UTF-8?q?Capitalise:=20"pyodide"=20=E2=9E=A1?=
=?UTF-8?q?=EF=B8=8F=20"Pyodide"?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Co-Authored-By: Hood Chatham <roberthoodchatham@gmail.com>
---
test/test_emscripten.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/test_emscripten.py b/test/test_emscripten.py
index aa7750bc5..7a8fd9953 100644
--- a/test/test_emscripten.py
+++ b/test/test_emscripten.py
@@ -20,7 +20,7 @@
def check_node():
# cibuildwheel adds a pinned node version to the PATH
- # check it's in the PATH then, check it's the one that runs pyodide
+ # check it's in the PATH then, check it's the one that runs Pyodide
cibw_cache_path = Path(sys.argv[1]).resolve(strict=True)
# find the node executable in PATH
node = shutil.which("node")
From 321e0dee3b7facc42fa8e913c56a96a810c9817a Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 20:21:26 +0530
Subject: [PATCH 43/51] =?UTF-8?q?"work"=20=E2=9E=A1=EF=B8=8F=20"may=20succ?=
=?UTF-8?q?eed"?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Co-Authored-By: Hood Chatham <roberthoodchatham@gmail.com>
---
docs/options.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/options.md b/docs/options.md
index 8281836ef..d8cff279c 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -255,7 +255,7 @@ Default: `auto`
- For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows.
- For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows.
-- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts work, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information.
+- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information.
This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config.
From 92babdb76184f1b5b29b310ea223c7cbef49ef36 Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 20:28:00 +0530
Subject: [PATCH 44/51] Add another job to test a custom Pyodide version
---
.github/workflows/test.yml | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3757be634..45ba25b09 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -171,9 +171,13 @@ jobs:
run: uv run pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py
test-pyodide:
- name: Test cibuildwheel building pyodide wheels
+ name: Test cibuildwheel building Pyodide wheels (${{ matrix.pyodide-version }} version)
needs: lint
runs-on: ubuntu-24.04
+ strategy:
+ fail-fast: false
+ matrix:
+ pyodide-version: ["default", "custom"]
timeout-minutes: 180
steps:
- uses: actions/checkout@v4
@@ -191,12 +195,26 @@ jobs:
uv run -m test.test_projects test.test_0_basic.basic_project sample_proj
- name: Run a sample build (GitHub Action)
+ if: matrix.pyodide-version == 'default'
+ uses: ./
+ with:
+ package-dir: sample_proj
+ output-dir: wheelhouse
+ env:
+ CIBW_PLATFORM: pyodide
+
+ - name: Run a sample build (GitHub Action) for an overridden Pyodide version
+ if: matrix.pyodide-version == 'custom'
uses: ./
with:
package-dir: sample_proj
output-dir: wheelhouse
+ # In case this breaks at any point in time, switch to using the latest version
+ # available or any other version that is not the same as the default one set
+ # in cibuildwheel/resources/build-platforms.toml.
env:
CIBW_PLATFORM: pyodide
+ CIBW_PYODIDE_VERSION: "v0.27.0a2"
- name: Run tests with 'CIBW_PLATFORM' set to 'pyodide'
run: |
From d73f71f8c812971d98205e76dae40325157d7baa Mon Sep 17 00:00:00 2001
From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>
Date: Thu, 21 Nov 2024 21:32:02 +0530
Subject: [PATCH 45/51] Handle "v"-prefixed + non-prefixed versions
---
.github/workflows/test.yml | 2 +-
cibuildwheel/pyodide.py | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 45ba25b09..6320821a7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -214,7 +214,7 @@ jobs:
# in cibuildwheel/resources/build-platforms.toml.
env:
CIBW_PLATFORM: pyodide
- CIBW_PYODIDE_VERSION: "v0.27.0a2"
+ CIBW_PYODIDE_VERSION: "0.27.0a2"
- name: Run tests with 'CIBW_PLATFORM' set to 'pyodide'
run: |
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 6fb4447ae..b8f688094 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -103,7 +103,7 @@ def validate_xbuildenv_version(
f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
f" version {pyodide_build_version}. The compatible versions available to download are:\n"
f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to"
- f" find the compatible versions for {pyodide_build_version}."
+ f" find the compatible versions for 'pyodide-build' {pyodide_build_version}."
)
raise errors.FatalError(msg)
@@ -228,6 +228,8 @@ def setup_python(
cibw_pyodide_version = os.environ.get(
"CIBW_PYODIDE_VERSION", python_configuration.pyodide_version
)
+ # If there's a "v" prefix, remove it: both would be equally valid
+ cibw_pyodide_version = cibw_pyodide_version.lstrip("v")
log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...")
# Search for compatible xbuildenv versions
compatible_versions = get_xbuildenv_versions(env)
From 1b4f911c1118eb60627e95bbc59f97093293e9dc Mon Sep 17 00:00:00 2001
From: Joe Rickerby <joerick@mac.com>
Date: Fri, 21 Mar 2025 18:26:30 +0000
Subject: [PATCH 46/51] Convert to a proper toml-able option, and remove some
hardcoded versions
This removes the enscripten and pyodide-build version specs from
pyproject.toml - pyodide-build is spec'd in the constraints file, and
the emscripten version can be read from the pyodide-build output.
---
cibuildwheel/options.py | 4 +
cibuildwheel/pyodide.py | 179 ++++++++++++--------
cibuildwheel/resources/build-platforms.toml | 2 +-
cibuildwheel/resources/defaults.toml | 2 +
cibuildwheel/util/helpers.py | 15 ++
unit_test/utils_test.py | 32 ++++
6 files changed, 161 insertions(+), 73 deletions(-)
diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py
index fa7605343..287f1d02f 100644
--- a/cibuildwheel/options.py
+++ b/cibuildwheel/options.py
@@ -108,6 +108,7 @@ class BuildOptions:
build_frontend: BuildFrontendConfig | None
config_settings: str
container_engine: OCIContainerEngineConfig
+ pyodide_version: str | None
@property
def package_dir(self) -> Path:
@@ -825,6 +826,8 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions:
msg = f"Failed to parse container config. {e}"
raise errors.ConfigurationError(msg) from e
+ pyodide_version = self.reader.get("pyodide-version", env_plat=False)
+
return BuildOptions(
globals=self.globals,
test_command=test_command,
@@ -844,6 +847,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions:
build_frontend=build_frontend,
config_settings=config_settings,
container_engine=container_engine,
+ pyodide_version=pyodide_version or None,
)
def check_for_invalid_configuration(self, identifiers: Iterable[str]) -> None:
diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py
index 8598a309e..97918e38e 100644
--- a/cibuildwheel/pyodide.py
+++ b/cibuildwheel/pyodide.py
@@ -4,11 +4,12 @@
import shutil
import sys
import tomllib
+import typing
from collections.abc import Sequence, Set
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
-from typing import Final
+from typing import Final, TypedDict
from filelock import FileLock
@@ -30,7 +31,7 @@
extract_zip,
move_file,
)
-from .util.helpers import prepare_command
+from .util.helpers import prepare_command, unwrap, unwrap_preserving_paragraphs
from .util.packaging import combine_constraints, find_compatible_wheel, get_pip_version
from .venv import virtualenv
@@ -41,12 +42,23 @@
class PythonConfiguration:
version: str
identifier: str
- pyodide_version: str
- pyodide_build_version: str
- emscripten_version: str
+ default_pyodide_version: str
node_version: str
+class PyodideXBuildEnvInfoVersionRange(TypedDict):
+ min: str | None
+ max: str | None
+
+
+class PyodideXBuildEnvInfo(TypedDict):
+ version: str
+ python: str
+ emscripten: str
+ pyodide_build: PyodideXBuildEnvInfoVersionRange
+ compatible: bool
+
+
@functools.cache
def ensure_node(major_version: str) -> Path:
with resources.NODEJS.open("rb") as f:
@@ -79,8 +91,6 @@ def ensure_node(major_version: str) -> Path:
def install_emscripten(tmp: Path, version: str) -> Path:
- # We don't need to match the emsdk version to the version we install, but
- # we do for stability
url = f"https://github.com/emscripten-core/emsdk/archive/refs/tags/{version}.zip"
installation_path = CIBW_CACHE_PATH / f"emsdk-{version}"
emsdk_path = installation_path / f"emsdk-{version}/emsdk"
@@ -98,44 +108,70 @@ def install_emscripten(tmp: Path, version: str) -> Path:
return emcc_path
-def get_xbuildenv_versions(env: dict[str, str]) -> list[str]:
- """Searches for the compatible xbuildenvs for the current pyodide-build version"""
- with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
- xbuildenvs = call(
- "pyodide",
- "xbuildenv",
- "search",
- "--json",
- "--all",
- env=env,
- cwd=CIBW_CACHE_PATH,
- capture_stdout=True,
- ).strip()
- xbuildenvs_dict = json.loads(xbuildenvs)
- compatible_xbuildenv_versions = [
- _["version"] for _ in xbuildenvs_dict["environments"] if _["compatible"]
- ]
-
- return compatible_xbuildenv_versions
-
-
-# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
-# which will always be compatible. Hence, this condition really checks only for the case where the
-# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an
-# error as appropriate.
-def validate_xbuildenv_version(
- cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str]
+def get_all_xbuildenv_version_info(env: dict[str, str]) -> list[PyodideXBuildEnvInfo]:
+ xbuildenvs_info_str = call(
+ "pyodide",
+ "xbuildenv",
+ "search",
+ "--json",
+ "--all",
+ env=env,
+ cwd=CIBW_CACHE_PATH,
+ capture_stdout=True,
+ ).strip()
+
+ xbuildenvs_info = json.loads(xbuildenvs_info_str)
+
+ if "environments" not in xbuildenvs_info:
+ msg = f"Invalid xbuildenvs info, got {xbuildenvs_info}"
+ raise ValueError(msg)
+
+ return typing.cast(list[PyodideXBuildEnvInfo], xbuildenvs_info["environments"])
+
+
+def get_xbuildenv_version_info(
+ env: dict[str, str], version: str, pyodide_build_version: str
+) -> PyodideXBuildEnvInfo:
+ xbuildenvs_info = get_all_xbuildenv_version_info(env)
+ for xbuildenv_info in xbuildenvs_info:
+ if xbuildenv_info["version"] == version:
+ return xbuildenv_info
+
+ msg = unwrap(f"""
+ Could not find pyodide xbuildenv version {version} in the available
+ versions as reported by pyodide-build v{pyodide_build_version}.
+ Available pyodide xbuild versions are:
+ {", ".join(e["version"] for e in xbuildenvs_info if e["compatible"])}
+ """)
+ raise errors.FatalError(msg)
+
+
+# The xbuildenv version is brought in sync with the pyodide-build version in
+# the constraints file, which will always be compatible with the version in
+# build-platforms.toml. Hence, this condition really checks only for the case
+# where the version is supplied manually through a CIBW_PYODIDE_VERSION
+# environment variable and raises an error as appropriate.
+def validate_pyodide_build_version(
+ xbuildenv_info: PyodideXBuildEnvInfo, pyodide_build_version: str
) -> None:
- """Validate the Pyodide version if set manually for the current pyodide-build version
- against a list of compatible versions."""
-
- if cibw_pyodide_version not in compatible_versions:
- msg = (
- f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
- f" version {pyodide_build_version}. The compatible versions available to download are:\n"
- f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to"
- f" find the compatible versions for 'pyodide-build' {pyodide_build_version}."
- )
+ """
+ Validate the Pyodide version is compatible with the installed
+ pyodide-build version.
+ """
+
+ pyodide_version = xbuildenv_info["version"]
+
+ if not xbuildenv_info["compatible"]:
+ msg = unwrap_preserving_paragraphs(f"""
+ The Pyodide xbuildenv version {pyodide_version} is not compatible
+ with the pyodide-build version {pyodide_build_version}. Please use
+ the 'pyodide xbuildenv search --all' command to find a compatible
+ version.
+
+ Set the pyodide-build version using the `dependency-constraints`
+ option, or set the Pyodide xbuildenv version using the
+ `pyodide-version` option.
+ """)
raise errors.FatalError(msg)
@@ -192,8 +228,10 @@ def setup_python(
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
environment: ParsedEnvironment,
+ user_pyodide_version: str | None,
) -> dict[str, str]:
base_python = get_base_python(python_configuration.identifier)
+ pyodide_version = user_pyodide_version or python_configuration.default_pyodide_version
log.step("Setting up build environment...")
venv_path = tmp / "venv"
@@ -247,30 +285,28 @@ def setup_python(
env=env,
)
- log.step(f"Installing Emscripten version: {python_configuration.emscripten_version} ...")
- emcc_path = install_emscripten(tmp, python_configuration.emscripten_version)
+ pyodide_build_version = call(
+ "python",
+ "-c",
+ "from importlib.metadata import version; print(version('pyodide-build'))",
+ env=env,
+ capture_stdout=True,
+ ).strip()
+
+ xbuildenv_info = get_xbuildenv_version_info(env, pyodide_version, pyodide_build_version)
+ validate_pyodide_build_version(
+ xbuildenv_info=xbuildenv_info,
+ pyodide_build_version=pyodide_build_version,
+ )
+
+ enscripten_version = xbuildenv_info["emscripten"]
+ log.step(f"Installing Emscripten version: {enscripten_version} ...")
+ emcc_path = install_emscripten(tmp, enscripten_version)
env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
- # Allow overriding the xbuildenv version with an environment variable. This allows
- # testing new Pyodide xbuildenv versions before they are officially released, or for
- # using a different Pyodide version other than the one that is listed to be compatible
- # with the pyodide-build version in the build-platforms.toml file.
- cibw_pyodide_version = os.environ.get(
- "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version
- )
- # If there's a "v" prefix, remove it: both would be equally valid
- cibw_pyodide_version = cibw_pyodide_version.lstrip("v")
- log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...")
- # Search for compatible xbuildenv versions
- compatible_versions = get_xbuildenv_versions(env)
- # and then validate the xbuildenv version
- validate_xbuildenv_version(
- cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions
- )
- env["PYODIDE_ROOT"] = install_xbuildenv(
- env, python_configuration.pyodide_build_version, cibw_pyodide_version
- )
+ log.step(f"Installing Pyodide xbuildenv version: {pyodide_version} ...")
+ env["PYODIDE_ROOT"] = install_xbuildenv(env, pyodide_build_version, pyodide_version)
return env
@@ -318,9 +354,7 @@ def build(options: Options, tmp_path: Path) -> None:
log.build_start(config.identifier)
- # Include both the identifier and the Pyodide version in the temp directory name
- cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", config.pyodide_version)
- identifier_tmp_dir = tmp_path / f"{config.identifier}_{cibw_pyodide_version}"
+ identifier_tmp_dir = tmp_path / config.identifier
built_wheel_dir = identifier_tmp_dir / "built_wheel"
repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
@@ -336,10 +370,11 @@ def build(options: Options, tmp_path: Path) -> None:
)
env = setup_python(
- identifier_tmp_dir / "build",
- config,
- dependency_constraint_flags,
- build_options.environment,
+ tmp=identifier_tmp_dir / "build",
+ python_configuration=config,
+ dependency_constraint_flags=dependency_constraint_flags,
+ environment=build_options.environment,
+ user_pyodide_version=build_options.pyodide_version,
)
pip_version = get_pip_version(env)
# The Pyodide command line runner mounts all directories in the host
diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml
index b5fa071a2..5803ef98d 100644
--- a/cibuildwheel/resources/build-platforms.toml
+++ b/cibuildwheel/resources/build-platforms.toml
@@ -161,7 +161,7 @@ python_configurations = [
[pyodide]
python_configurations = [
- { identifier = "cp312-pyodide_wasm32", version = "3.12", pyodide_version = "0.27.0", pyodide_build_version = "0.29.2", emscripten_version = "3.1.58", node_version = "v20" },
+ { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.0", node_version = "v22" },
]
[ios]
diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml
index 6b65e6369..3a77055fa 100644
--- a/cibuildwheel/resources/defaults.toml
+++ b/cibuildwheel/resources/defaults.toml
@@ -25,6 +25,8 @@ test-groups = []
container-engine = "docker"
+pyodide-version = ""
+
manylinux-x86_64-image = "manylinux2014"
manylinux-i686-image = "manylinux2014"
manylinux-aarch64-image = "manylinux2014"
diff --git a/cibuildwheel/util/helpers.py b/cibuildwheel/util/helpers.py
index dbef452c0..363fd3345 100644
--- a/cibuildwheel/util/helpers.py
+++ b/cibuildwheel/util/helpers.py
@@ -76,6 +76,21 @@ def unwrap(text: str) -> str:
return re.sub(r"\s+", " ", text)
+def unwrap_preserving_paragraphs(text: str) -> str:
+ """
+ Unwraps multi-line text to a single line, but preserves paragraphs
+ """
+ # remove initial line indent
+ text = textwrap.dedent(text)
+ # remove leading/trailing whitespace
+ text = text.strip()
+
+ paragraphs = text.split("\n\n")
+ # remove consecutive whitespace
+ paragraphs = [re.sub(r"\s+", " ", paragraph) for paragraph in paragraphs]
+ return "\n\n".join(paragraphs)
+
+
def parse_key_value_string(
key_value_string: str,
positional_arg_names: Sequence[str] | None = None,
diff --git a/unit_test/utils_test.py b/unit_test/utils_test.py
index 3c4a58859..8b0be33d3 100644
--- a/unit_test/utils_test.py
+++ b/unit_test/utils_test.py
@@ -12,6 +12,8 @@
format_safe,
parse_key_value_string,
prepare_command,
+ unwrap,
+ unwrap_preserving_paragraphs,
)
from cibuildwheel.util.packaging import find_compatible_wheel
@@ -363,3 +365,33 @@ def test_copy_test_sources_alternate_copy_into(sample_project):
],
any_order=True,
)
+
+
+def test_unwrap():
+ assert (
+ unwrap("""
+ This is a
+ multiline
+ string
+ """)
+ == "This is a multiline string"
+ )
+
+
+def test_unwrap_preserving_paragraphs():
+ assert (
+ unwrap("""
+ This is a
+ multiline
+ string
+ """)
+ == "This is a multiline string"
+ )
+ assert (
+ unwrap_preserving_paragraphs("""
+ paragraph one
+
+ paragraph two
+ """)
+ == "paragraph one\n\nparagraph two"
+ )
From 9d0f6bc63410aa5117aa7800af54995713bcdadb Mon Sep 17 00:00:00 2001
From: Joe Rickerby <joerick@mac.com>
Date: Fri, 21 Mar 2025 19:45:37 +0000
Subject: [PATCH 47/51] Add a schema entry
---
bin/generate_schema.py | 3 ++
.../resources/cibuildwheel.schema.json | 36 +++++++++++++++----
2 files changed, 33 insertions(+), 6 deletions(-)
diff --git a/bin/generate_schema.py b/bin/generate_schema.py
index b82a91136..776869a09 100755
--- a/bin/generate_schema.py
+++ b/bin/generate_schema.py
@@ -181,6 +181,9 @@
musllinux-x86_64-image:
type: string
description: Specify alternative manylinux / musllinux container images
+ pyodide-version:
+ type: string
+ description: Specify the version of Pyodide to use
repair-wheel-command:
type: string_array
description: Execute a shell command to repair each built wheel.
diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json
index 46af911c8..3da3af1f8 100644
--- a/cibuildwheel/resources/cibuildwheel.schema.json
+++ b/cibuildwheel/resources/cibuildwheel.schema.json
@@ -397,6 +397,11 @@
"description": "Specify alternative manylinux / musllinux container images",
"title": "CIBW_MUSLLINUX_X86_64_IMAGE"
},
+ "pyodide-version": {
+ "type": "string",
+ "description": "Specify the version of Pyodide to use",
+ "title": "CIBW_PYODIDE_VERSION"
+ },
"repair-wheel-command": {
"description": "Execute a shell command to repair each built wheel.",
"oneOf": [
@@ -658,6 +663,9 @@
"musllinux-x86_64-image": {
"$ref": "#/properties/musllinux-x86_64-image"
},
+ "pyodide-version": {
+ "$ref": "#/properties/pyodide-version"
+ },
"repair-wheel-command": {
"$ref": "#/properties/repair-wheel-command"
},
@@ -758,6 +766,9 @@
"musllinux-x86_64-image": {
"$ref": "#/properties/musllinux-x86_64-image"
},
+ "pyodide-version": {
+ "$ref": "#/properties/pyodide-version"
+ },
"repair-wheel-command": {
"description": "Execute a shell command to repair each built wheel.",
"oneOf": [
@@ -822,6 +833,9 @@
"environment": {
"$ref": "#/properties/environment"
},
+ "pyodide-version": {
+ "$ref": "#/properties/pyodide-version"
+ },
"repair-wheel-command": {
"$ref": "#/properties/repair-wheel-command"
},
@@ -873,6 +887,9 @@
"environment": {
"$ref": "#/properties/environment"
},
+ "pyodide-version": {
+ "$ref": "#/properties/pyodide-version"
+ },
"repair-wheel-command": {
"description": "Execute a shell command to repair each built wheel.",
"oneOf": [
@@ -937,6 +954,9 @@
"environment": {
"$ref": "#/properties/environment"
},
+ "pyodide-version": {
+ "$ref": "#/properties/pyodide-version"
+ },
"repair-wheel-command": {
"$ref": "#/properties/repair-wheel-command"
},
@@ -954,11 +974,6 @@
},
"test-requires": {
"$ref": "#/properties/test-requires"
- },
- "pyodide-version": {
- "description": "Specify the Pyodide xbuildenv version to use for building",
- "type": "string",
- "title": "CIBW_PYODIDE_VERSION"
}
}
},
@@ -993,6 +1008,9 @@
"environment": {
"$ref": "#/properties/environment"
},
+ "pyodide-version": {
+ "$ref": "#/properties/pyodide-version"
+ },
"repair-wheel-command": {
"$ref": "#/properties/repair-wheel-command"
},
@@ -1002,10 +1020,16 @@
"test-extras": {
"$ref": "#/properties/test-extras"
},
+ "test-sources": {
+ "$ref": "#/properties/test-sources"
+ },
+ "test-groups": {
+ "$ref": "#/properties/test-groups"
+ },
"test-requires": {
"$ref": "#/properties/test-requires"
}
- }
+ }
}
}
}
From b9f55e1ec5e601debe78776e1c90b89ac59143da Mon Sep 17 00:00:00 2001
From: Joe Rickerby <joerick@mac.com>
Date: Mon, 24 Mar 2025 21:50:18 +0000
Subject: [PATCH 48/51] Add docs for CIBW_PYODIDE_VERSION
---
docs/options.md | 39 +++++++++++++++++++++++++++++++++++++++
docs/platforms/pyodide.md | 7 ++-----
2 files changed, 41 insertions(+), 5 deletions(-)
diff --git a/docs/options.md b/docs/options.md
index 53a1ad27d..59010f31f 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -1431,6 +1431,45 @@ Platform-specific environment variables are also available:<br/>
dependency-versions = { packages = ["pyodide-build==0.29.1"] }
```
+### `CIBW_PYODIDE_VERSION` {: #pyodide-version}
+
+> Specify the Pyodide version to use for `pyodide` platform builds
+
+This option allows you to specify a specific version of Pyodide to be used when building wheels for the `pyodide` platform. By default, cibuildwheel will use a default Pyodide version compatible with the current cibuildwheel and `pyodide-build` versions.
+
+This option is particularly useful for:
+
+- Testing against specific Pyodide alpha or older releases.
+- Ensuring reproducibility by targeting a known Pyodide version.
+
+The available Pyodide versions are determined by the version of `pyodide-build` being used. You can list the compatible versions using the command `pyodide xbuildenv search --all` as described in the [Pyodide platform documentation](platforms/pyodide.md#choosing-a-version).
+
+!!! tip
+ You can set the version of `pyodide-build` using the [`CIBW_DEPENDENCY_VERSIONS`](#dependency-versions) option.
+
+#### Examples
+
+!!! tab examples "Environment variables"
+
+ ```yaml
+ # Build Pyodide wheels using Pyodide version 0.26.4
+ CIBW_PYODIDE_VERSION: 0.26.4
+
+ # Build Pyodide wheels using a specific alpha release
+ CIBW_PYODIDE_VERSION: 0.27.0a2
+ ```
+
+!!! tab examples "pyproject.toml"
+
+ ```toml
+ [tool.cibuildwheel.pyodide]
+ # Build Pyodide wheels using Pyodide version 0.26.4
+ pyodide-version = "0.26.4"
+
+ [tool.cibuildwheel.pyodide]
+ # Build Pyodide wheels using a specific alpha release
+ pyodide-version = "0.27.0a2"
+ ```
## Testing
diff --git a/docs/platforms/pyodide.md b/docs/platforms/pyodide.md
index 38a5afc70..672dbbd69 100644
--- a/docs/platforms/pyodide.md
+++ b/docs/platforms/pyodide.md
@@ -12,9 +12,6 @@ You need to have a matching host version of Python (unlike all other cibuildwhee
You must target pyodide with `--platform pyodide` (or use `--only` on the identifier).
-!!!tip
+## Choosing a Pyodide version {: #choosing-a-version}
- It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` environment variable to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json) or can be
- referenced using the `pyodide xbuildenv search` command.
-
- This option is **not available** in the `pyproject.toml` config.
+It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` option to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json), which can be viewed more easily by installing `pyodide-build` from PyPI and use `pyodide xbuildenv search --all` to see a compatibility table.
From 611b032dcd7866de6ca9a3dbb017bfc4625e9eac Mon Sep 17 00:00:00 2001
From: Joe Rickerby <joerick@mac.com>
Date: Mon, 24 Mar 2025 21:54:35 +0000
Subject: [PATCH 49/51] Rephrase
---
docs/options.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/options.md b/docs/options.md
index 59010f31f..56c86c2dc 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -1435,7 +1435,7 @@ Platform-specific environment variables are also available:<br/>
> Specify the Pyodide version to use for `pyodide` platform builds
-This option allows you to specify a specific version of Pyodide to be used when building wheels for the `pyodide` platform. By default, cibuildwheel will use a default Pyodide version compatible with the current cibuildwheel and `pyodide-build` versions.
+This option allows you to specify a specific version of Pyodide to be used when building wheels for the `pyodide` platform. If unset, cibuildwheel will use a pinned Pyodide version.
This option is particularly useful for:
From 16f4c5d947ca2a2f498d6969311ee072e862b989 Mon Sep 17 00:00:00 2001
From: Joe Rickerby <joerick@mac.com>
Date: Mon, 24 Mar 2025 22:09:42 +0000
Subject: [PATCH 50/51] Add tests for pyodide-version
---
test/test_emscripten.py | 28 ++++++++++++++++++++++++++++
unit_test/options_test.py | 5 +++++
2 files changed, 33 insertions(+)
diff --git a/test/test_emscripten.py b/test/test_emscripten.py
index 897cd67f1..87a97516c 100644
--- a/test/test_emscripten.py
+++ b/test/test_emscripten.py
@@ -1,4 +1,5 @@
import shutil
+import subprocess
import sys
import textwrap
@@ -84,3 +85,30 @@ def test_pyodide_build(tmp_path, use_pyproject_toml):
print("expected_wheels", expected_wheels)
assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_pyodide_version_incompatible(tmp_path, capfd):
+ if sys.platform == "win32":
+ pytest.skip("emsdk doesn't work correctly on Windows")
+
+ if not shutil.which("python3.12"):
+ pytest.skip("Python 3.12 not installed")
+
+ if detect_ci_provider() == CIProvider.travis_ci:
+ pytest.skip("Python 3.12 is just a non-working pyenv shim")
+
+ basic_project.generate(tmp_path)
+
+ with pytest.raises(subprocess.CalledProcessError):
+ utils.cibuildwheel_run(
+ tmp_path,
+ add_args=["--platform", "pyodide"],
+ add_env={
+ "CIBW_DEPENDENCY_VERSIONS": "packages: pyodide-build==0.29.3",
+ "CIBW_PYODIDE_VERSION": "0.26.0a6",
+ },
+ )
+
+ out, err = capfd.readouterr()
+
+ assert "is not compatible with the pyodide-build version" in err
diff --git a/unit_test/options_test.py b/unit_test/options_test.py
index 499862a99..55384f149 100644
--- a/unit_test/options_test.py
+++ b/unit_test/options_test.py
@@ -30,6 +30,8 @@
environment-pass = ["EXAMPLE_ENV"]
+pyodide-version = "0.26.4"
+
[tool.cibuildwheel.macos]
test-requires = "else"
@@ -84,6 +86,9 @@ def test_options_1(tmp_path, monkeypatch):
assert local.test_sources == ["test", "other dir"]
assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux2014"]
+ local = options.build_options("cp312-pyodide_wasm32")
+ assert local.pyodide_version == "0.26.4"
+
def test_passthrough(tmp_path, monkeypatch):
with tmp_path.joinpath("pyproject.toml").open("w") as f:
From aef35c9a01e8bcf86fad14c9f7d55281a420e592 Mon Sep 17 00:00:00 2001
From: Joe Rickerby <joerick@mac.com>
Date: Wed, 26 Mar 2025 13:56:50 +0000
Subject: [PATCH 51/51] Apply suggestions from code review
---
docs/options.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/options.md b/docs/options.md
index 32a225525..3a4670b83 100644
--- a/docs/options.md
+++ b/docs/options.md
@@ -256,7 +256,7 @@ Default: `auto`
- For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows.
- For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows.
- For `ios` you need to be running on macOS, with Xcode and the iOS simulator installed.
-- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information.
+- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](platforms/pyodide.md) for more information.
This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config.
@@ -308,7 +308,7 @@ The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425]
Windows arm64 platform support is experimental.
For an experimental WebAssembly build with `--platform pyodide`,
-`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.26.4`](https://github.com/pyodide/pyodide/releases/tag/0.26.4).
+`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.27.0`](https://github.com/pyodide/pyodide/releases/tag/0.27.0).
See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7.