Skip to content

Commit 2183c59

Browse files
mayeutmayeut
committedAug 19, 2023
chore: rework PyPy installation
PyPy installation is just a tar extract. As such, it might be interesting to install PyPy at runtime rather than at build time. This is especially true for EOL versions of PyPy which would allow to reduce the image size, and thus, overall build time for users not using EOL PyPy versions. This commit does not modify default installed PyPy versions yet. This commit will also be useful for GraalPy installation.
1 parent 4434c7d commit 2183c59

12 files changed

+508
-183
lines changed
 

‎docker/Dockerfile

+20-32
Original file line numberDiff line numberDiff line change
@@ -138,43 +138,31 @@ FROM build_cpython AS build_cpython312
138138
COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
139139
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.0rc1
140140

141-
FROM build_cpython AS all_python
142-
COPY build_scripts/install-pypy.sh \
143-
build_scripts/pypy.sha256 \
144-
build_scripts/finalize-python.sh \
145-
/build_scripts/
146-
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9
147-
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.11
148-
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.12
149-
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.10 7.3.12
150-
COPY --from=build_cpython36 /opt/_internal /opt/_internal/
151-
COPY --from=build_cpython37 /opt/_internal /opt/_internal/
152-
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
153-
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
154-
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
155-
COPY --from=build_cpython311 /opt/_internal /opt/_internal/
156-
COPY --from=build_cpython312 /opt/_internal /opt/_internal/
157-
RUN manylinux-entrypoint /build_scripts/finalize-python.sh
141+
FROM build_cpython AS all_cpython
142+
COPY build_scripts/finalize-python.sh /build_scripts/
143+
RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
144+
--mount=type=bind,target=/build_cpython37,from=build_cpython37 \
145+
--mount=type=bind,target=/build_cpython38,from=build_cpython38 \
146+
--mount=type=bind,target=/build_cpython39,from=build_cpython39 \
147+
--mount=type=bind,target=/build_cpython310,from=build_cpython310 \
148+
--mount=type=bind,target=/build_cpython311,from=build_cpython311 \
149+
--mount=type=bind,target=/build_cpython312,from=build_cpython312 \
150+
mkdir -p /opt/_internal && \
151+
cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
152+
manylinux-entrypoint /build_scripts/finalize-python.sh
158153

159154

160155
FROM runtime_base
161156
COPY --from=build_git /manylinux-rootfs /
162157
COPY --from=build_cpython /manylinux-rootfs /
163-
COPY --from=all_python /opt/_internal /opt/_internal/
164-
COPY build_scripts/finalize.sh \
165-
build_scripts/update-system-packages.sh \
166-
build_scripts/python-tag-abi-tag.py \
167-
build_scripts/requirements3.6.txt \
168-
build_scripts/requirements3.7.txt \
169-
build_scripts/requirements3.8.txt \
170-
build_scripts/requirements3.9.txt \
171-
build_scripts/requirements3.10.txt \
172-
build_scripts/requirements3.11.txt \
173-
build_scripts/requirements3.12.txt \
174-
build_scripts/requirements-base-tools.txt \
175-
/build_scripts/
176-
COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
177-
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
158+
COPY build_scripts /opt/_internal/build_scripts/
159+
RUN --mount=type=bind,target=/all_cpython,from=all_cpython \
160+
cp -rf /all_cpython/opt/_internal/* /opt/_internal/ && \
161+
manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \
162+
pp37-pypy37_pp73 \
163+
pp38-pypy38_pp73 \
164+
pp39-pypy39_pp73 \
165+
pp310-pypy310_pp73
178166

179167
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
180168

‎docker/build_scripts/download-and-install-interpreter.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
# Stop at any error, show all commands
4+
set -exuo pipefail
5+
6+
# Get script directory
7+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
8+
9+
ABI_TAG=$1
10+
DOWNLOAD_URL=$2
11+
SHA256=$3
12+
13+
PREFIX="/opt/_internal/${ABI_TAG}"
14+
15+
case ${DOWNLOAD_URL} in
16+
*.tar) COMP=;;
17+
*.tar.gz) COMP=z;;
18+
*.tar.bz2) COMP=j;;
19+
*.tar.xz) COMP=J;;
20+
*) echo "unsupported archive"; exit 1;;
21+
esac
22+
23+
mkdir ${PREFIX}
24+
25+
curl -fsSL ${DOWNLOAD_URL} | tee >(tar -C ${PREFIX} --strip-components 1 -x${COMP}f -) | sha256sum -c <(echo "${SHA256} -")
26+
27+
# remove debug symbols if any
28+
find ${PREFIX}/bin -name '*.debug' -delete
29+
30+
${MY_DIR}/finalize-one.sh ${PREFIX}

‎docker/build_scripts/finalize-one.sh

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
3+
# Stop at any error, show all commands
4+
set -exuo pipefail
5+
6+
PREFIX=$1
7+
8+
# Get script directory
9+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
10+
11+
# Some python's install as bin/python3. Make them available as
12+
# bin/python.
13+
if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
14+
ln -s python3 ${PREFIX}/bin/python
15+
fi
16+
PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
17+
PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)")
18+
19+
# Install pinned packages for this python version.
20+
# Use the already intsalled cpython pip to bootstrap pip if available
21+
if [ -f /usr/local/bin/python${PY_VER} ]; then
22+
/usr/local/bin/python${PY_VER} -m pip --python ${PREFIX}/bin/python install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
23+
else
24+
${PREFIX}/bin/python -m ensurepip
25+
${PREFIX}/bin/python -m pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
26+
fi
27+
if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
28+
ln -s pip3 ${PREFIX}/bin/pip
29+
fi
30+
# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
31+
ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
32+
ln -s ${PREFIX} /opt/python/${ABI_TAG}
33+
# Make versioned python commands available directly in environment.
34+
if [[ "${PY_IMPL}" == "cpython" ]]; then
35+
ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
36+
fi
37+
ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}

‎docker/build_scripts/finalize.sh

+14-22
Original file line numberDiff line numberDiff line change
@@ -11,30 +11,21 @@ source $MY_DIR/build_utils.sh
1111

1212
mkdir /opt/python
1313
for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do
14-
# Some python's install as bin/python3. Make them available as
15-
# bin/python.
16-
if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
17-
ln -s python3 ${PREFIX}/bin/python
18-
fi
19-
${PREFIX}/bin/python -m ensurepip
20-
if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
21-
ln -s pip3 ${PREFIX}/bin/pip
22-
fi
23-
PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
24-
# Since we fall back on a canned copy of pip, we might not have
25-
# the latest pip and friends. Upgrade them to make sure.
26-
${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
27-
# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
28-
ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
29-
ln -s ${PREFIX} /opt/python/${ABI_TAG}
30-
# Make versioned python commands available directly in environment.
31-
if [[ "${PREFIX}" == *"/pypy"* ]]; then
32-
ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}
33-
else
34-
ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
35-
fi
14+
${MY_DIR}/finalize-one.sh ${PREFIX}
3615
done
3716

17+
# create manylinux-interpreters script
18+
cat <<EOF > /usr/local/bin/manylinux-interpreters
19+
#!/bin/bash
20+
21+
set -euo pipefail
22+
23+
/opt/python/cp310-cp310/bin/python $MY_DIR/manylinux-interpreters.py "\$@"
24+
EOF
25+
chmod 755 /usr/local/bin/manylinux-interpreters
26+
27+
MANYLINUX_INTERPRETERS_NO_CHECK=1 /usr/local/bin/manylinux-interpreters ensure "$@"
28+
3829
# Create venv for auditwheel & certifi
3930
TOOLS_PATH=/opt/_internal/tools
4031
/opt/python/cp310-cp310/bin/python -m venv $TOOLS_PATH
@@ -84,6 +75,7 @@ clean_pyc /opt/_internal
8475

8576
# remove cache
8677
rm -rf /root/.cache
78+
rm -rf /tmp/* || true
8779

8880
hardlink -cv /opt/_internal
8981

‎docker/build_scripts/install-pypy.sh

-72
This file was deleted.

0 commit comments

Comments
 (0)