From 651d32e90f2ca3a136fc4aeeb2e5677b8d0c464b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sat, 1 Aug 2026 21:58:15 +0200 Subject: [PATCH 1/3] fix(tests): probe / instead of /api for OpenShift port-forward readiness RHAIENG-6036 (08f4fe374) already fixed this correctly: probe / with a persistent session (cookies are required across RStudio's multi-hop redirect chain), because RStudio doesn't support /api the way Jupyter and code-server do. The hermetic-codeserver backport (da9540c47) later silently reverted this hunk back to a bare 30s/localhost/no-session probe, and a same-day follow-up fix (74873588f) only noticed the regressed timeout, re-fixing it by copying main's /api-only probe -- but main dropped RStudio entirely, so that path never worked here. Confirmed against real ghcr.io rhoai-2.25 images: RStudio's nginx redirects /api to a hardcoded http://127.0.0.1:8888/api/, which is unreachable through a port-forward where the local port isn't literally 8888. / works for rstudio, jupyter-minimal, and codeserver alike, but only with a redirect-following session (a bare GET redirect-loops on RStudio's auth flow without persisted cookies). See red-hat-data-services/notebooks#2684 for the CI failure history this caused (rstudio/cuda-rstudio amd64 timing out on nearly every push). Co-Authored-By: Claude Sonnet 5 --- tests/containers/kubernetes_utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/containers/kubernetes_utils.py b/tests/containers/kubernetes_utils.py index 33a0b9a528..60c2b62540 100644 --- a/tests/containers/kubernetes_utils.py +++ b/tests/containers/kubernetes_utils.py @@ -267,12 +267,22 @@ def deploy( self.port = p.get_actual_port() LOGGER.debug(f"Listening on port {self.port}") - # Use 120s timeout for slower cold starts (e.g. code-server on arm64). + + # Use TIMEOUT_2MIN for slower cold starts (e.g. code-server on arm64). + # Always probe / (not /api): RStudio's nginx redirects /api to a + # hardcoded http://127.0.0.1:8888/api/, which is wrong (and unreachable) + # through a port-forward where the local port isn't literally 8888. + # https://github.com/red-hat-data-services/notebooks/issues/2684 + def _ready() -> bool: + with requests.Session() as session: + response = session.get(f"http://127.0.0.1:{self.port}/", timeout=5, allow_redirects=True) + return response.status_code == 200 + Wait.until( "Connecting to pod succeeds", 1, - 120, - lambda: requests.get(f"http://127.0.0.1:{self.port}/api", timeout=10).status_code == 200, + TestFrameConstants.TIMEOUT_2MIN, + _ready, ) LOGGER.debug("Done setting up portforward") From c342da836479bf2a35315409334e2696f541df5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sat, 1 Aug 2026 22:03:56 +0200 Subject: [PATCH 2/3] ci(test-containers): add a job to run the openshift-marked container tests The existing self-test workflow only runs `pytest tests/containers -m 'not openshift and not cuda and not rocm'`, explicitly skipping the OpenShift/Kubernetes deployment path (ImageDeployment / test_image_run_on_openshift). That's exactly the code path that silently regressed and went unnoticed for weeks (see #2684, #2685) -- nothing in this repo's CI actually exercised it outside of the full image build workflow. Add openshift-container-tests, reusing find-images' matrix, mirroring the provisioning steps build-notebooks-TEMPLATE.yaml already uses for the same purpose: rootful podman (install-podman-action, sharing image storage with cri-o) + a kubeadm cluster (provision-k8s), then `pytest -m 'openshift and not cuda and not rocm'` against the pulled image. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/test-containers.yaml | 65 ++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/.github/workflows/test-containers.yaml b/.github/workflows/test-containers.yaml index 65a96d2961..9486a52666 100644 --- a/.github/workflows/test-containers.yaml +++ b/.github/workflows/test-containers.yaml @@ -158,3 +158,68 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: junit.xml + + openshift-container-tests: + name: "openshift: ${{ matrix.name }}" + needs: find-images + runs-on: ubuntu-26.04 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.find-images.outputs.matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version-file: pyproject.toml + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install deps + run: uv sync --locked + + # Rootful podman sharing image storage with cri-o, same as build-notebooks-TEMPLATE.yaml, + # so the pod created by the openshift-marked tests can find the image without re-pulling. + - name: Install and configure Podman + uses: './.github/actions/install-podman-action' + with: + platform: linux/amd64 + + - name: Provision K8s cluster + uses: ./.github/actions/provision-k8s + + - name: Login to GHCR + uses: docker/login-action@371161bbe7024a29a25c5e19bfcbc0804fe9ad2c # v4.5.2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull and run OpenShift container tests + env: + TEST_IMAGE: ${{ matrix.image }} + DOCKER_HOST: "unix:///var/run/podman/podman.sock" + TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/podman/podman.sock" + run: | + set -Eeuxo pipefail + podman pull "${TEST_IMAGE}" + uv run pytest --capture=fd tests/containers \ + -m 'openshift and not cuda and not rocm' \ + --image="${TEST_IMAGE}" \ + --junitxml=junit-openshift.xml \ + -v + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1.2.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: junit-openshift.xml From a624f5a7475803f153a6e890e32833244799fb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sat, 1 Aug 2026 22:17:08 +0200 Subject: [PATCH 3/3] fix(ci): disable zstd:chunked partial pulls in openshift-container-tests install-podman-action's storage.conf enables partial-image pulls (enable_partial_images = "true"), which install-podman-action's own callers rely on for speed when podman build pulls base images. But this job instead `podman pull`s an already-published ghcr.io image, and ghcr.io's blob endpoint returns "501 Unsupported client range" for the resulting ranged fetch, failing the pull outright (observed on 3 consecutive CI runs of PR #2685): Error: unable to copy from source docker://ghcr.io/.../codeserver...: partial pull of blob sha256:...: read zstd:chunked manifest: fetching partial blob: received unexpected HTTP status: 501 Unsupported client range Disable it for this job only (not the shared ci/cached-builds/storage.conf, which other build jobs still benefit from) by patching /etc/containers/ storage.conf after install-podman-action runs. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/test-containers.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test-containers.yaml b/.github/workflows/test-containers.yaml index 9486a52666..0bc32a1f55 100644 --- a/.github/workflows/test-containers.yaml +++ b/.github/workflows/test-containers.yaml @@ -203,6 +203,14 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # install-podman-action's storage.conf enables zstd:chunked partial pulls, which + # only matters for `podman build`'s own registry pulls elsewhere. Here we `podman + # pull` an already-published ghcr.io image, and ghcr.io's blob endpoint returns + # "501 Unsupported client range" for partial/ranged fetches, failing the pull + # outright instead of falling back to a full one. Disable it for this job only. + - name: Disable zstd:chunked partial image pulls + run: sudo sed -i 's/enable_partial_images = "true"/enable_partial_images = "false"/' /etc/containers/storage.conf + - name: Pull and run OpenShift container tests env: TEST_IMAGE: ${{ matrix.image }}