chore: rework PyPy installation

PyPy installation is just a tar extract.
As such, it might be interesting to install PyPy at runtime rather than at build time.
This is especially true for EOL versions of PyPy which would allow to reduce the image size,
and thus, overall build time for users not using EOL PyPy versions.

This commit does not modify default installed PyPy versions yet.

This commit will also be useful for GraalPy installation.

docker/Dockerfile

@@ -138,43 +138,31 @@ FROM build_cpython AS build_cpython312
 COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
 RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.0rc1
 
-FROM build_cpython AS all_python
-COPY build_scripts/install-pypy.sh \
-     build_scripts/pypy.sha256 \
-     build_scripts/finalize-python.sh \
-     /build_scripts/
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.11
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.12
-RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.10 7.3.12
-COPY --from=build_cpython36 /opt/_internal /opt/_internal/
-COPY --from=build_cpython37 /opt/_internal /opt/_internal/
-COPY --from=build_cpython38 /opt/_internal /opt/_internal/
-COPY --from=build_cpython39 /opt/_internal /opt/_internal/
-COPY --from=build_cpython310 /opt/_internal /opt/_internal/
-COPY --from=build_cpython311 /opt/_internal /opt/_internal/
-COPY --from=build_cpython312 /opt/_internal /opt/_internal/
-RUN manylinux-entrypoint /build_scripts/finalize-python.sh
+FROM build_cpython AS all_cpython
+COPY build_scripts/finalize-python.sh /build_scripts/
+RUN --mount=type=bind,target=/build_cpython36,from=build_cpython36 \
+    --mount=type=bind,target=/build_cpython37,from=build_cpython37 \
+    --mount=type=bind,target=/build_cpython38,from=build_cpython38 \
+    --mount=type=bind,target=/build_cpython39,from=build_cpython39 \
+    --mount=type=bind,target=/build_cpython310,from=build_cpython310 \
+    --mount=type=bind,target=/build_cpython311,from=build_cpython311 \
+    --mount=type=bind,target=/build_cpython312,from=build_cpython312 \
+    mkdir -p /opt/_internal && \
+    cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
+    manylinux-entrypoint /build_scripts/finalize-python.sh
 
 
 FROM runtime_base
 COPY --from=build_git /manylinux-rootfs /
 COPY --from=build_cpython /manylinux-rootfs /
-COPY --from=all_python /opt/_internal /opt/_internal/
-COPY build_scripts/finalize.sh \
-     build_scripts/update-system-packages.sh \
-     build_scripts/python-tag-abi-tag.py \
-     build_scripts/requirements3.6.txt \
-     build_scripts/requirements3.7.txt \
-     build_scripts/requirements3.8.txt \
-     build_scripts/requirements3.9.txt \
-     build_scripts/requirements3.10.txt \
-     build_scripts/requirements3.11.txt \
-     build_scripts/requirements3.12.txt \
-     build_scripts/requirements-base-tools.txt \
-     /build_scripts/
-COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
-RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
+COPY build_scripts /opt/_internal/build_scripts/
+RUN --mount=type=bind,target=/all_cpython,from=all_cpython \
+    cp -rf /all_cpython/opt/_internal/* /opt/_internal/ && \
+    manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \
+    pp37-pypy37_pp73 \
+    pp38-pypy38_pp73 \
+    pp39-pypy39_pp73 \
+    pp310-pypy310_pp73
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 

docker/build_scripts/download-and-install-interpreter.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+ABI_TAG=$1
+DOWNLOAD_URL=$2
+SHA256=$3
+
+PREFIX="/opt/_internal/${ABI_TAG}"
+
+case ${DOWNLOAD_URL} in
+  *.tar) COMP=;;
+  *.tar.gz) COMP=z;;
+  *.tar.bz2) COMP=j;;
+  *.tar.xz) COMP=J;;
+  *) echo "unsupported archive"; exit 1;;
+esac
+
+mkdir ${PREFIX}
+
+curl -fsSL ${DOWNLOAD_URL} | tee >(tar -C ${PREFIX} --strip-components 1 -x${COMP}f -) | sha256sum -c <(echo "${SHA256} -")
+
+# remove debug symbols if any
+find ${PREFIX}/bin -name '*.debug' -delete
+
+${MY_DIR}/finalize-one.sh ${PREFIX}

docker/build_scripts/finalize-one.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+PREFIX=$1
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# Some python's install as bin/python3. Make them available as
+# bin/python.
+if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
+	ln -s python3 ${PREFIX}/bin/python
+fi
+PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
+PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)")
+
+# Install pinned packages for this python version.
+# Use the already intsalled cpython pip to bootstrap pip if available
+if [ -f /usr/local/bin/python${PY_VER} ]; then
+	/usr/local/bin/python${PY_VER} -m pip --python ${PREFIX}/bin/python install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
+else
+	${PREFIX}/bin/python -m ensurepip
+	${PREFIX}/bin/python -m pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
+fi
+if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
+	ln -s pip3 ${PREFIX}/bin/pip
+fi
+# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
+ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
+ln -s ${PREFIX} /opt/python/${ABI_TAG}
+# Make versioned python commands available directly in environment.
+if [[ "${PY_IMPL}" == "cpython" ]]; then
+	ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
+fi
+ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER}

docker/build_scripts/finalize.sh

@@ -11,30 +11,21 @@ source $MY_DIR/build_utils.sh
 
 mkdir /opt/python
 for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do
-	# Some python's install as bin/python3. Make them available as
-	# bin/python.
-	if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
-		ln -s python3 ${PREFIX}/bin/python
-	fi
-	${PREFIX}/bin/python -m ensurepip
-	if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
-		ln -s pip3 ${PREFIX}/bin/pip
-	fi
-	PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))")
-	# Since we fall back on a canned copy of pip, we might not have
-	# the latest pip and friends. Upgrade them to make sure.
-	${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt
-	# Create a symlink to PREFIX using the ABI_TAG in /opt/python/
-	ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py)
-	ln -s ${PREFIX} /opt/python/${ABI_TAG}
-	# Make versioned python commands available directly in environment.
-	if [[ "${PREFIX}" == *"/pypy"* ]]; then
-		ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}
-	else
-		ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}
-	fi
+	${MY_DIR}/finalize-one.sh ${PREFIX}
 done
 
+# create manylinux-interpreters script
+cat <<EOF > /usr/local/bin/manylinux-interpreters
+#!/bin/bash
+
+set -euo pipefail
+
+/opt/python/cp310-cp310/bin/python $MY_DIR/manylinux-interpreters.py "\$@"
+EOF
+chmod 755 /usr/local/bin/manylinux-interpreters
+
+MANYLINUX_INTERPRETERS_NO_CHECK=1 /usr/local/bin/manylinux-interpreters ensure "$@"
+
 # Create venv for auditwheel & certifi
 TOOLS_PATH=/opt/_internal/tools
 /opt/python/cp310-cp310/bin/python -m venv $TOOLS_PATH
@@ -84,6 +75,7 @@ clean_pyc /opt/_internal
 
 # remove cache
 rm -rf /root/.cache
+rm -rf /tmp/* || true
 
 hardlink -cv /opt/_internal