diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb988e71..21ec34f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,21 +152,29 @@ jobs: integration-tests: strategy: - fail-fast: false - matrix: - app-path: [ 'datafusion', 'hudi-file-group-api/cpp', 'hudi-table-api/rust' , 'hudi-table-api/python' ] + fail-fast: false + matrix: + app-path: [ 'datafusion', 'hudi-file-group-api/cpp', 'hudi-table-api/rust', 'hudi-table-api/python' ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + + - name: Pull pre-built runner image + run: | + docker pull andreab92/hudi-demo-runner:latest + docker tag andreab92/hudi-demo-runner:latest demo-runner:latest + - name: Pre-pull base images run: | docker pull quay.io/minio/minio:latest + - name: Prepare cache directories with correct permissions run: | mkdir -p .cargo/registry .cargo/git target python/target .uv-cache sudo chown -R $(id -u):$(id -g) .cargo target python/target .uv-cache + - name: Cache integration build artifacts (cargo/target/uv) uses: actions/cache@v5 with: @@ -176,13 +184,16 @@ jobs: .cargo/git python/target .uv-cache - key: integration-${{ runner.os }}-${{ hashFiles('**/Cargo.lock', 'python/Cargo.toml', 'demo/infra/runner/Dockerfile') }} + key: integration-${{ runner.os }}-${{ matrix.app-path }}-${{ hashFiles('**/Cargo.lock', 'python/Cargo.toml', 'demo/infra/runner/Dockerfile') }} restore-keys: | + integration-${{ runner.os }}-${{ matrix.app-path }}- integration-${{ runner.os }}- + - name: Integration test - ${{ matrix.app-path }} run: | cd demo ./ci_run.sh ${{ matrix.app-path }} + - name: Fix cache ownership before save if: always() run: | diff --git a/demo/compose.yaml b/demo/compose.yaml index 2becd436..52f2c271 100644 --- a/demo/compose.yaml +++ b/demo/compose.yaml @@ -73,3 +73,4 @@ services: networks: demo_network: driver: bridge + diff --git a/demo/infra/runner/Dockerfile b/demo/infra/runner/Dockerfile index c649feef..d9e8eaeb 100644 --- a/demo/infra/runner/Dockerfile +++ b/demo/infra/runner/Dockerfile @@ -19,12 +19,13 @@ FROM rust:1.87 RUN apt-get update && apt-get install -y cmake curl ca-certificates +# Install uv to a globally accessible location (not /root/.local/bin) +# This allows the container to run as any user (HOST_UID:HOST_GID) ADD https://astral.sh/uv/install.sh /uv-installer.sh +RUN UV_INSTALL_DIR=/usr/local/bin sh /uv-installer.sh && rm /uv-installer.sh -RUN sh /uv-installer.sh && rm /uv-installer.sh - -ENV PATH="/root/.local/bin/:$PATH" - +# Install Python to a shared location accessible by any user +ENV UV_PYTHON_INSTALL_DIR=/opt/python RUN uv python install 3.13 RUN uv python list @@ -36,6 +37,10 @@ ENV PATH="/opt/.venv/bin:$PATH" RUN uv pip install --only-binary=all 'pyarrow==20.0.0' +# Make Python installation and venv accessible to any user (needed for HOST_UID:HOST_GID mapping) +# Python needs read+execute, venv needs read+write+execute for pip install +RUN chmod -R a+rX /opt/python && chmod -R a+rwX /opt/.venv + WORKDIR /opt CMD tail -f /dev/null