Skip to content

Commit cd3b37f

Browse files
mayeutmayeut
committed
chore: rework PyPy installation
PyPy installation is just a tar extract. As such, it might be interesting to install PyPy at runtime rather than at build time. This is especially true for EOL versions of PyPy which would allow to reduce the image size, and thus, overall build time for users not using EOL PyPy versions. This commit does not modify default installed PyPy versions yet. This commit will also be useful for GraalPy installation.
1 parent 0828984 commit cd3b37f

13 files changed

+555
-184
lines changed

‎README.rst

+44-1
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ Image content
173173

174174
All images currently contain:
175175

176-
- CPython 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and PyPy 3.7, 3.8, 3.9 installed in
176+
- CPython 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and PyPy 3.7, 3.8, 3.9, 3.10 installed in
177177
``/opt/python/<python tag>-<abi tag>``. The directories are named
178178
after the PEP 425 tags for each environment --
179179
e.g. ``/opt/python/cp37-cp37m`` contains a CPython 3.7 build, and
@@ -184,6 +184,49 @@ All images currently contain:
184184

185185
- The `auditwheel <https://pypi.python.org/pypi/auditwheel>`_ tool
186186

187+
- The manylinux-interpreters tool which allows to list all available interpreters & install ones missing from the image
188+
189+
3 commands are available:
190+
191+
- ``manylinux-interpreters list``
192+
193+
.. code-block:: bash
194+
195+
usage: manylinux-interpreters list [-h] [-v] [-i] [--format {text,json}]
196+
197+
list available or installed interpreters
198+
199+
options:
200+
-h, --help show this help message and exit
201+
-v, --verbose display additional information (--format=text only, ignored for --format=json)
202+
-i, --installed only list installed interpreters
203+
--format {text,json} text is not meant to be machine readable (i.e. the format is not stable)
204+
205+
- ``manylinux-interpreters ensure-all``
206+
207+
.. code-block:: bash
208+
209+
usage: manylinux-interpreters ensure-all [-h]
210+
211+
make sure all interpreters are installed
212+
213+
options:
214+
-h, --help show this help message and exit
215+
216+
- ``manylinux-interpreters ensure``
217+
218+
.. code-block:: bash
219+
220+
usage: manylinux-interpreters ensure [-h] TAG [TAG ...]
221+
222+
make sure a list of interpreters are installed
223+
224+
positional arguments:
225+
TAG tag with format '<python tag>-<abi tag>' e.g. 'pp310-pypy310_pp73'
226+
227+
options:
228+
-h, --help show this help message and exit
229+
187230
Note that less common or virtually unheard of flag combinations
188231
(such as ``--with-pydebug`` (``d``) and ``--without-pymalloc`` (absence of ``m``)) are not provided.
189232

‎docker/Dockerfile

+20-32
Original file line numberDiff line numberDiff line change
@@ -138,43 +138,31 @@ FROM build_cpython AS build_cpython312
138138
COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
139139
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.0rc1
140140

141-
FROM build_cpython AS all_python
142-
COPY build_scripts/install-pypy.sh \
143-
build_scripts/pypy.sha256 \
144-
build_scripts/finalize-python.sh \
145-
/build_scripts/
146-
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9
147-
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.11
148-
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.12
149-
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.10 7.3.12
150-
COPY --from=build_cpython36 /opt/_internal /opt/_internal/
151-
COPY --from=build_cpython37 /opt/_internal /opt/_internal/
152-
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
153-
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
154-
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
155-
COPY --from=build_cpython311 /opt/_internal /opt/_internal/
156-
COPY --from=build_cpython312 /opt/_internal /opt/_internal/
157-
RUN manylinux-entrypoint /build_scripts/finalize-python.sh
141+
FROM build_cpython AS all_cpython
142+
COPY build_scripts/finalize-python.sh /build_scripts/
143+
RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
144+
--mount=type=bind,target=/build_cpython37,from=build_cpython37 \
145+
--mount=type=bind,target=/build_cpython38,from=build_cpython38 \
146+
--mount=type=bind,target=/build_cpython39,from=build_cpython39 \
147+
--mount=type=bind,target=/build_cpython310,from=build_cpython310 \
148+
--mount=type=bind,target=/build_cpython311,from=build_cpython311 \
149+
--mount=type=bind,target=/build_cpython312,from=build_cpython312 \
150+
mkdir -p /opt/_internal && \
151+
cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
152+
manylinux-entrypoint /build_scripts/finalize-python.sh
158153

159154

160155
FROM runtime_base
161156
COPY --from=build_git /manylinux-rootfs /
162157
COPY --from=build_cpython /manylinux-rootfs /
163-
COPY --from=all_python /opt/_internal /opt/_internal/
164-
COPY build_scripts/finalize.sh \
165-
build_scripts/update-system-packages.sh \
166-
build_scripts/python-tag-abi-tag.py \
167-
build_scripts/requirements3.6.txt \
168-
build_scripts/requirements3.7.txt \
169-
build_scripts/requirements3.8.txt \
170-
build_scripts/requirements3.9.txt \
171-
build_scripts/requirements3.10.txt \
172-
build_scripts/requirements3.11.txt \
173-
build_scripts/requirements3.12.txt \
174-
build_scripts/requirements-base-tools.txt \
175-
/build_scripts/
176-
COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
177-
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
158+
COPY build_scripts /opt/_internal/build_scripts/
159+
RUN --mount=type=bind,target=/all_cpython,from=all_cpython \
160+
cp -rf /all_cpython/opt/_internal/* /opt/_internal/ && \
161+
manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \
162+
pp37-pypy37_pp73 \
163+
pp38-pypy38_pp73 \
164+
pp39-pypy39_pp73 \
165+
pp310-pypy310_pp73
178166

179167
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
180168

‎docker/build_scripts/download-and-install-interpreter.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
# Stop at any error, show all commands
4+
set -exuo pipefail
5+
6+
# Get script directory
7+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
8+
9+
ABI_TAG=$1
10+
DOWNLOAD_URL=$2
11+
SHA256=$3
12+
13+
PREFIX="/opt/_internal/${ABI_TAG}"
14+
15+
case ${DOWNLOAD_URL} in
16+
*.tar) COMP=;;
17+
*.tar.gz) COMP=z;;
18+
*.tar.bz2) COMP=j;;
19+
*.tar.xz) COMP=J;;
20+
*) echo "unsupported archive"; exit 1;;
21+
esac
22+
23+
mkdir ${PREFIX}
24+
25+
curl -fsSL ${DOWNLOAD_URL} | tee >(tar -C ${PREFIX} --strip-components 1 -x${COMP}f -) | sha256sum -c <(echo "${SHA256} -")
26+
27+
# remove debug symbols if any
28+
find ${PREFIX}/bin -name '*.debug' -delete
29+
30+
${MY_DIR}/finalize-one.sh ${PREFIX}

‎docker/build_scripts/finalize-one.sh

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
3+
# Stop at any error, show all commands
4+
set -exuo pipefail
5+
6+
PREFIX=$1
7+
8+
# Get script directory
9+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
10+
11+
# Some python's install as bin/python3. Make them available as
12+
# bin/python.
13+
if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
14+
ln -s python3 ${PREFIX}/bin/python
15+
fi
16+
PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
17+
PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)")
18+
19+
# Install pinned packages for this python version.
20+
# Use the already intsalled cpython pip to bootstrap pip if available
21+
if [ -f /usr/local/bin/python${PY_VER} ]; then
22+
/usr/local/bin/python${PY_VER} -m pip --python ${PREFIX}/bin/python install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
23+
else
24+
${PREFIX}/bin/python -m ensurepip
25+
${PREFIX}/bin/python -m pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
26+
fi
27+
if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
28+
ln -s pip3 ${PREFIX}/bin/pip
29+
fi
30+
# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
31+
ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
32+
ln -s ${PREFIX} /opt/python/${ABI_TAG}
33+
# Make versioned python commands available directly in environment.
34+
if [[ "${PY_IMPL}" == "cpython" ]]; then
35+
ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
36+
fi
37+
ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}

‎docker/build_scripts/finalize.sh

+14-22
Original file line numberDiff line numberDiff line change
@@ -11,30 +11,21 @@ source $MY_DIR/build_utils.sh
1111

1212
mkdir /opt/python
1313
for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do
14-
# Some python's install as bin/python3. Make them available as
15-
# bin/python.
16-
if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
17-
ln -s python3 ${PREFIX}/bin/python
18-
fi
19-
${PREFIX}/bin/python -m ensurepip
20-
if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
21-
ln -s pip3 ${PREFIX}/bin/pip
22-
fi
23-
PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
24-
# Since we fall back on a canned copy of pip, we might not have
25-
# the latest pip and friends. Upgrade them to make sure.
26-
${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
27-
# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
28-
ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
29-
ln -s ${PREFIX} /opt/python/${ABI_TAG}
30-
# Make versioned python commands available directly in environment.
31-
if [[ "${PREFIX}" == *"/pypy"* ]]; then
32-
ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}
33-
else
34-
ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
35-
fi
14+
${MY_DIR}/finalize-one.sh ${PREFIX}
3615
done
3716

17+
# create manylinux-interpreters script
18+
cat <<EOF > /usr/local/bin/manylinux-interpreters
19+
#!/bin/bash
20+
21+
set -euo pipefail
22+
23+
/opt/python/cp310-cp310/bin/python $MY_DIR/manylinux-interpreters.py "\$@"
24+
EOF
25+
chmod 755 /usr/local/bin/manylinux-interpreters
26+
27+
MANYLINUX_INTERPRETERS_NO_CHECK=1 /usr/local/bin/manylinux-interpreters ensure "$@"
28+
3829
# Create venv for auditwheel & certifi
3930
TOOLS_PATH=/opt/_internal/tools
4031
/opt/python/cp310-cp310/bin/python -m venv $TOOLS_PATH
@@ -84,6 +75,7 @@ clean_pyc /opt/_internal
8475

8576
# remove cache
8677
rm -rf /root/.cache
78+
rm -rf /tmp/* || true
8779

8880
hardlink -cv /opt/_internal
8981

‎docker/build_scripts/install-pypy.sh

-72
This file was deleted.

0 commit comments

Comments
 (0)