[rhoai-2.25] RHAIENG-2645: add loop to retry package installation when it fails

jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu

@@ -1,4 +1,4 @@
 #########################
 # configuration args    #
 #########################
 ARG BASE_IMAGE
@@ -68,6 +68,7 @@
 # upgrade first to avoid fixable vulnerabilities end
 
 # Install useful OS packages
+COPY jupyter/utils utils/
 RUN --mount=type=cache,target=/var/cache/dnf /bin/bash <<'EOF'
 set -Eeuxo pipefail
 echo "Building for architecture: ${TARGETARCH}"
@@ -80,7 +81,7 @@
     PACKAGES=(perl mesa-libGL skopeo)
 fi
 echo "Installing: ${PACKAGES[*]}"
-dnf install -y "${PACKAGES[@]}"
+./utils/install_with_retry.sh dnf-install "${PACKAGES[@]}"
 dnf clean all && rm -rf /var/cache/yum
 EOF
 
@@ -163,8 +164,7 @@
     # Install build dependencies (shared for pyarrow and onnx)
     # ninja-build is required by scikit-build-core (pyarrow's PEP 517 build backend) to drive
     # the CMake build; `python -m build` fails with "Missing dependencies: ninja>=1.5" without it.
-    dnf install -y cmake make gcc-c++ pybind11-devel wget ninja-build
-    dnf clean all
+    /opt/app-root/bin/utils/install_with_retry.sh dnf-install cmake make gcc-c++ pybind11-devel wget ninja-build
     # Build and collect pyarrow wheel
     git clone --depth 1 --branch "${ARROW_BRANCH}" https://github.com/apache/arrow.git
     cd arrow/cpp
@@ -235,8 +235,7 @@
 RUN /bin/bash <<'EOF'
 set -Eeuxo pipefail
 if [ "${TARGETARCH}" = "ppc64le" ]; then
-    dnf install -y gcc-toolset-13 cmake ninja-build git wget unzip
-    dnf clean all
+    /opt/app-root/bin/utils/install_with_retry.sh dnf-install gcc-toolset-13 cmake ninja-build git wget unzip
 else
     echo "Skipping common-builder package install on non-Power"
 fi
@@ -307,7 +306,8 @@
 USER 0
 
 # Dependencies for PDF export begin
-RUN ./utils/install_pdf_deps.sh
+RUN ./utils/install_with_retry.sh texlive-install
+ENV PATH="/usr/local/texlive/bin/linux:/usr/local/pandoc/bin:$PATH"
 # Dependencies for PDF export end
 
 USER 1001
@@ -341,11 +341,7 @@
 USER root
 
 # Install useful OS packages
-RUN /bin/bash <<'EOF'
-set -Eeuxo pipefail
-dnf install -y jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat
-dnf clean all && rm -rf /var/cache/yum
-EOF
+RUN ./utils/install_with_retry.sh dnf-install jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat
 
 # Copy dynamically-linked mongocli built in earlier build stage
 COPY --from=mongocli-builder /tmp/mongocli /opt/app-root/bin/

jupyter/utils/install_with_retry.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# Shared install script with retry logic for dnf, texlive (install_pdf_deps), and npm.
+# Usage:
+#   ./install_with_retry.sh dnf-install <package> [package ...]
+#   ./install_with_retry.sh texlive-install
+#   ./install_with_retry.sh npm-install
+
+set -Eeuxo pipefail
+
+readonly MAX_RETRIES="${MAX_RETRIES:-3}"
+readonly RETRY_DELAY="${RETRY_DELAY:-30}"
+
+# Runs a command with retry logic.
+# Optional: set CLEANUP_CMD to a shell command (e.g. "dnf clean metadata") to run between retries.
+# Returns 0 on success, exits 1 after MAX_RETRIES failures.
+run_with_retry() {
+    local retry_count=0
+    while true; do
+        if "$@"; then
+            return 0
+        fi
+        retry_count=$((retry_count + 1))
+        if [ "$retry_count" -ge "$MAX_RETRIES" ]; then
+            echo "ERROR: Command failed after $MAX_RETRIES attempts" >&2
+            exit 1
+        fi
+        echo "Command failed (attempt $retry_count/$MAX_RETRIES), retrying in ${RETRY_DELAY} seconds..."
+        if [ -n "${CLEANUP_CMD:-}" ]; then
+            $CLEANUP_CMD || true
+        fi
+        sleep "$RETRY_DELAY"
+    done
+}
+
+dnf_install() {
+    local packages=("$@")
+    if [ ${#packages[@]} -eq 0 ]; then
+        echo "Usage: $0 dnf-install <package> [package ...]" >&2
+        exit 1
+    fi
+    CLEANUP_CMD="dnf clean metadata" run_with_retry dnf install -y ${DNF_EXTRA_OPTS:-} "${packages[@]}"
+    dnf clean all
+    rm -rf /var/cache/yum
+}
+
+texlive_install() {
+    local script_dir
+    script_dir="$(cd "$(dirname "$0")" && pwd)"
+    CLEANUP_CMD= run_with_retry "$script_dir/install_pdf_deps.sh"
+}
+
+npm_install() {
+    CLEANUP_CMD="rm -rf node_modules" run_with_retry npm install
+}
+
+main() {
+    case "${1:-}" in
+        dnf-install)
+            shift
+            dnf_install "$@"
+            ;;
+        texlive-install)
+            texlive_install
+            ;;
+        npm-install)
+            npm_install
+            ;;
+        *)
+            echo "Usage: $0 {dnf-install|texlive-install|npm-install} [args...]" >&2
+            echo "  dnf-install <package> [package ...]  Install RPM packages with retry" >&2
+            echo "  texlive-install                       Run install_pdf_deps.sh with retry" >&2
+            echo "  npm-install                           Run npm install with retry" >&2
+            exit 1
+            ;;
+    esac
+}
+
+# Only run main when this script is executed directly (not when sourced by run-code-server.sh)
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main "$@"
+fi