ci: OPS-724: Move to ARC runners (#2904)

Signed-off-by: Dillon Cullinan <[email protected]>
Signed-off-by: hongkuanz <[email protected]>

Author: dillon-cullinan

.github/workflows/container-validation-backends.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   build-test:
-    runs-on: gpu-l40-runners
+    runs-on: gpu-l40-amd64
     strategy:
       matrix:
         framework: [vllm]
@@ -45,16 +45,13 @@ jobs:
         if: always()
         run: |
           docker system prune -af
-      - name: Debug
-        run: |
-          lsmod | grep nvidia
-          sudo dmesg | grep -i nvrm || true
-          nvidia-smi
       - name: Build image
         env:
           GITHUB_TOKEN: ${{ secrets.CI_TOKEN }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           SCCACHE_S3_BUCKET:  ${{ secrets.SCCACHE_S3_BUCKET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           ./container/build.sh --tag ${{ matrix.framework }}:latest \
             --target ${{ matrix.target }} \
@@ -64,7 +61,8 @@ jobs:
             --sccache-region "$AWS_DEFAULT_REGION"
       - name: Run pytest
         run: |
-          docker run --rm --gpus all -w /workspace \
+          docker run --runtime=nvidia --rm --gpus all -w /workspace \
+            --network host \
             --name ${{ env.CONTAINER_ID }}_pytest \
             ${{ matrix.framework }}:latest \
             bash -c "pytest -xsv --basetemp=/tmp --junitxml=${{ env.PYTEST_XML_FILE }} -m \"${{ env.PYTEST_MARKS }}\""

container/Dockerfile

@@ -35,6 +35,8 @@ ARG ARCH_ALT=x86_64
 ARG USE_SCCACHE
 ARG SCCACHE_BUCKET=""
 ARG SCCACHE_REGION=""
+ARG AWS_ACCESS_KEY_ID=""
+ARG AWS_SECRET_ACCESS_KEY=""
 
 # NIXL configuration
 ARG NIXL_UCX_REF=v1.19.0
@@ -56,6 +58,8 @@ ARG PYTHON_VERSION
 ARG USE_SCCACHE
 ARG SCCACHE_BUCKET
 ARG SCCACHE_REGION
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
 ARG NIXL_UCX_REF
 ARG NIXL_REF
 
@@ -268,6 +272,8 @@ ARG ENABLE_KVBM
 ARG USE_SCCACHE
 ARG SCCACHE_BUCKET
 ARG SCCACHE_REGION
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
 
 WORKDIR /opt/dynamo
 

container/Dockerfile.vllm

@@ -20,6 +20,8 @@ ARG TORCH_BACKEND="cu128"
 ARG USE_SCCACHE
 ARG SCCACHE_BUCKET=""
 ARG SCCACHE_REGION=""
+ARG AWS_ACCESS_KEY_ID=""
+ARG AWS_SECRET_ACCESS_KEY=""
 
 # Match 0.10.1.1 vLLM release
 # https://github.com/vllm-project/vllm/releases/tag/v0.10.1.1
@@ -119,6 +121,8 @@ ARG USE_SCCACHE
 ARG ARCH_ALT
 ARG SCCACHE_BUCKET
 ARG SCCACHE_REGION
+ARG AWS_ACCESS_KEY_ID=""
+ARG AWS_SECRET_ACCESS_KEY=""
 
 ENV ARCH_ALT=${ARCH_ALT}
 RUN if [ "$USE_SCCACHE" = "true" ]; then \

container/build.sh

@@ -429,6 +429,10 @@ show_help() {
     echo "  [--use-sccache enable sccache for Rust/C/C++ compilation caching]"
     echo "  [--sccache-bucket S3 bucket name for sccache (required with --use-sccache)]"
     echo "  [--sccache-region S3 region for sccache (required with --use-sccache)]"
+    echo ""
+    echo "  Note: When using --use-sccache, AWS credentials must be set:"
+    echo "        export AWS_ACCESS_KEY_ID=your_access_key"
+    echo "        export AWS_SECRET_ACCESS_KEY=your_secret_key"
     exit 0
 }
 
@@ -598,6 +602,8 @@ if [ "$USE_SCCACHE" = true ]; then
     BUILD_ARGS+=" --build-arg USE_SCCACHE=true"
     BUILD_ARGS+=" --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}"
     BUILD_ARGS+=" --build-arg SCCACHE_REGION=${SCCACHE_REGION}"
+    BUILD_ARGS+=" --build-arg AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
+    BUILD_ARGS+=" --build-arg AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"
 fi
 
 LATEST_TAG="--tag dynamo:latest-${FRAMEWORK,,}"