Skip to content

Commit bd0667a

Browse files
mplanchardmplanchard
authored
MAINT: build manylinux wheels (#8)
* ENH: build manywheel wheels using official docker container * MAINT: improve error output in setup.py * TEST: run manylinux build without tests and on non-master * FIX: correct docker tag * FIX: remove ARM for now Running into the same issue as described here: nicolargo/glances#1479 * FIX: ensure clean build dir for python * FIX: more windows woes * CHORE: remove unused code * CHORE: what is wrong with windows now * MAINT: put build back after test * DOC: add comment to cargo.toml * TEST: bust the cache manually to see if it's caching problems * MAINT: more specific caching strategy
1 parent 42afba6 commit bd0667a

File tree

6 files changed

+150
-18
lines changed

6 files changed

+150
-18
lines changed

‎.github/workflows/ci.yml

+93-9
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,13 @@ jobs:
5151
~/.cargo/registry
5252
~/.cargo/git
5353
target
54-
# Rebuild whenever the cargo lock file changes
55-
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
54+
# Use the OS, the python version, and the hashed cargo lockfile as the
55+
# cache key. The Python version shouldn't be necessary, but I have
56+
# seen some weird failures in Windows CI where it gets the built
57+
# python targets confused. The Python version is included at the
58+
# end so it can be partially matched by cache keys in contexts
59+
# where we're not iterating over python envs.
60+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.python-version }}
5661

5762
# Cache `cargo install` built binaries
5863
- name: "Cache Built Binaries"
@@ -89,7 +94,9 @@ jobs:
8994
run: |
9095
cargo test --features=wasm
9196
make develop-py-wheel
97+
ls dist/*.whl
9298
pip install dist/*.whl
99+
echo "Running Tests"
93100
python tests/test_py.py
94101
env:
95102
WINDOWS: "${{ contains(runner.os, 'windows') }}"
@@ -139,7 +146,8 @@ jobs:
139146
~/.cargo/registry
140147
~/.cargo/git
141148
target
142-
# Rebuild whenever the cargo lock file changes
149+
# This should partial match the caches generated for the tests,
150+
# which include a python version at the end.
143151
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
144152

145153
# Cache `cargo install` built binaries
@@ -188,13 +196,84 @@ jobs:
188196
path: js/
189197
name: wasm-pkg
190198

191-
build-python-wheels:
192-
name: "Build Python Wheels"
199+
build-python-wheels-linux:
200+
# Linux wheels can't be built directly but instead need to be built
201+
# via a "manylinux" container as specified by PEPs 513, 571, and 599
202+
name: "Build Linux Wheels"
203+
needs: "test"
204+
if: "${{ github.ref == 'master' }}"
205+
runs-on: "ubuntu-latest"
206+
strategy:
207+
matrix:
208+
manylinux:
209+
- arch: "manylinux1_i686"
210+
img: "quay.io/pypa/manylinux1_i686:2020-07-04-283458f"
211+
- arch: "manylinux1_x86_64"
212+
img: "quay.io/pypa/manylinux1_x86_64:2020-07-04-283458f"
213+
- arch: "manylinux2010_i686"
214+
img: "quay.io/pypa/manylinux2010_i686:2020-07-04-10a3c30"
215+
- arch: "manylinux2010_x86_64"
216+
img: "quay.io/pypa/manylinux2010_x86_64:2020-07-04-10a3c30"
217+
- arch: "manylinux2014_i686"
218+
img: "quay.io/pypa/manylinux2014_i686:2020-07-04-bb5f087"
219+
- arch: "manylinux2014_x86_64"
220+
img: "quay.io/pypa/manylinux2014_x86_64:2020-07-04-bb5f087"
221+
steps:
222+
# Check out the code
223+
- uses: "actions/checkout@v2"
224+
# Set the current month and year (used for cache key)
225+
- name: "Get Date"
226+
id: get-date
227+
# Outputs e.g. "202007"
228+
# tbh I have yet to find the docs where this output format is
229+
# defined, but I copied this from the official cache action's README.
230+
run: |
231+
echo "::set-output name=date::$(/bin/date -u '+%Y%m')"
232+
shell: bash
233+
234+
# Generate the lockfile
235+
- name: "Generate Cargo Lockfile"
236+
run: "cargo generate-lockfile"
237+
238+
# Cache build dependencies
239+
- name: "Cache Build Fragments"
240+
id: "cache-build-fragments"
241+
uses: "actions/cache@v2"
242+
with:
243+
path: |
244+
~/.cargo/registry
245+
~/.cargo/git
246+
target
247+
# This should partial match the caches generated for the tests,
248+
# which include a python version at the end.
249+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
250+
251+
# Cache `cargo install` built binaries
252+
- name: "Cache Built Binaries"
253+
id: "cache-binaries"
254+
uses: "actions/cache@v2"
255+
with:
256+
path: "~/.cargo/bin"
257+
# In theory, this should rebuild binaries once a month
258+
key: "${{ runner.os }}-cargo-binaries-${{steps.get-date.outputs.date}}"
259+
260+
- name: "Build Wheels"
261+
run: "make build-py-wheel-manylinux"
262+
env:
263+
MANYLINUX_IMG: ${{ matrix.manylinux.img }}
264+
265+
- uses: "actions/upload-artifact@v2"
266+
with:
267+
path: "dist/*.whl"
268+
name: "py-${{ matrix.manylinux.arch }}-wheels"
269+
270+
build-python-wheels-mac-windows:
271+
name: "Build Python Wheels (MacOS, Windows)"
193272
needs: test
194273
if: "${{ github.ref == 'master' }}"
195274
strategy:
196275
matrix:
197-
os: [ubuntu-latest, macos-latest, windows-latest]
276+
os: [macos-latest, windows-latest]
198277
python-version: [3.6, 3.7, 3.8]
199278
runs-on: "${{ matrix.os }}"
200279
steps:
@@ -236,8 +315,13 @@ jobs:
236315
~/.cargo/registry
237316
~/.cargo/git
238317
target
239-
# Rebuild whenever the cargo lock file changes
240-
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
318+
# Use the OS, the python version, and the hashed cargo lockfile as the
319+
# cache key. The Python version shouldn't be necessary, but I have
320+
# seen some weird failures in Windows CI where it gets the built
321+
# python targets confused. The Python version is included at the
322+
# end so it can be partially matched by cache keys in contexts
323+
# where we're not iterating over python envs.
324+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.python-version }}
241325

242326
# Cache `cargo install` built binaries
243327
- name: "Cache Built Binaries"
@@ -264,4 +348,4 @@ jobs:
264348
- uses: "actions/upload-artifact@v2"
265349
with:
266350
path: "dist/*.whl"
267-
name: "py-${{ matrix.python-version }}-${{ runner.os }}-whl"
351+
name: "py-${{ matrix.python-version }}-${{ runner.os }}-wheels"

‎Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ edition = "2018"
77
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
88

99
[lib]
10+
# cdylib for CFFI and python integration
11+
# lib for regular rust stuff
1012
crate-type = ["cdylib", "lib"]
1113

1214
[features]

‎Makefile

+27-3
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,31 @@ build-wasm: setup
2626
debug-wasm:
2727
rm -rf ./js && wasm-pack build --target nodejs --out-dir js --out-name index --debug -- --features wasm
2828

29+
.PHONY: clean-py
30+
clean-py:
31+
rm -rf build/*
32+
rm -rf dist/*
33+
2934
.PHONY: build-py-sdist
30-
build-py-sdist: $(VENV)
35+
build-py-sdist: $(VENV) clean-py
3136
cargo clean -p jsonlogic
3237
$(VENV) setup.py sdist
3338

3439
.PHONY: build-py-wheel
35-
build-py-wheel: $(VENV)
40+
build-py-wheel: $(VENV) clean-py
3641
cargo clean -p jsonlogic
37-
rm -rf dist/*
3842
$(VENV) setup.py bdist_wheel
3943

44+
# NOTE: this command may sudo on linux
45+
.PHONY: build-py-wheel-manylinux
46+
build-py-wheel-manylinux: clean-py
47+
docker run -v "$$PWD":/io --rm $(MANYLINUX_IMG) /io/build-wheels.sh
48+
49+
.PHONY: build-py-all
50+
build-py-all: $(VENV) clean-py
51+
cargo clean -p jsonlogic
52+
$(VENV) setup.py sdist bdist_wheel
53+
4054
.PHONY: develop-py-wheel
4155
develop-py-wheel: $(VENV)
4256
$(VENV) setup.py bdist_wheel
@@ -45,6 +59,16 @@ develop-py-wheel: $(VENV)
4559
develop-py: $(VENV)
4660
$(VENV) setup.py develop
4761

62+
.PHONY: distribute-py
63+
distribute-py: $(VENV)
64+
$(VENV) -m pip install twine
65+
twine upload -s dist/*
66+
67+
.PHONY: test-distribute-py
68+
test-distribute-py:
69+
$(VENV) -m pip install twine
70+
twine upload -s --repository testpypi dist/*
71+
4872
.PHONY: setup
4973
setup:
5074
wasm-pack --version > /dev/null 2>&1 || cargo install wasm-pack

‎build-wheels.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
3+
# Expected to be run in a manylinux container
4+
5+
set -ex
6+
7+
cd /io
8+
9+
curl https://sh.rustup.rs -sSf | \
10+
sh -s -- --default-toolchain stable -y
11+
12+
export PATH=/root/.cargo/bin:$PATH
13+
14+
mkdir -p build && rm -rf build/*
15+
16+
for PYBIN in /opt/python/{cp36-cp36m,cp37-cp37m,cp38-cp38}/bin; do
17+
export PYTHON_SYS_EXECUTABLE="$PYBIN/python"
18+
19+
"${PYBIN}/python" -m ensurepip
20+
"${PYBIN}/python" -m pip install -U setuptools wheel setuptools-rust
21+
"${PYBIN}/python" setup.py bdist_wheel
22+
done
23+
24+
for whl in dist/*.whl; do
25+
auditwheel repair "$whl" -w dist/
26+
done

‎setup.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@
1313

1414
def get_version():
1515
proc = Popen(("cargo", "pkgid"), stdout=PIPE, stderr=PIPE)
16-
out, _err = tuple(map(bytes.decode, proc.communicate()))
16+
out, err = tuple(map(bytes.decode, proc.communicate()))
1717
if proc.returncode != 0:
18-
raise RuntimeError("Could not get Cargo package info")
18+
raise RuntimeError(f"Could not get Cargo package info: {err}")
1919
version = out.split(":")[-1]
2020
return version.strip()
2121

‎tests/test_py.rs

-4
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,6 @@
55
//! The actual tests are found in `test_py.py`. This file just serves
66
//! as a runner.
77
8-
#[cfg(feature = "python")]
9-
use std::fs;
10-
#[cfg(feature = "python")]
11-
use std::path;
128
#[cfg(feature = "python")]
139
use std::process::Command;
1410

0 commit comments

Comments
 (0)