[WIP] cuda13 test v3

.github/workflows/sycl-containers.yaml

@@ -51,6 +51,10 @@ jobs:
             file: ubuntu2404_build
             tag: latest
             build_args: ""
+          - name: Build Ubuntu 24.04 Docker image with CUDA 13.1
+            file: ubuntu2404_build_cuda131
+            tag: latest
+            build_args: ""
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -87,6 +91,9 @@ jobs:
           - name: Build + Intel Drivers Ubuntu 24.04 Docker image
             file: ubuntu2404_intel_drivers
             tag: latest
+          - name: Build + Intel Drivers Ubuntu 24.04 Docker image with CUDA 13.1
+            file: ubuntu2404_intel_drivers_cuda131
+            tag: latest
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

.github/workflows/sycl-linux-precommit.yml

@@ -159,6 +159,11 @@ jobs:
             runner: '["Linux", "cuda"]'
             image_options: -u 1001 --gpus all --cap-add SYS_ADMIN
             target_devices: cuda:gpu
+          - name: NVIDIA/CUDA 13.1
+            runner: '["Linux", "cuda13"]'
+            image: "ghcr.io/intel/llvm/ubuntu2404_intel_drivers_cuda131:latest"
+            image_options: -u 1001 --gpus all --cap-add SYS_ADMIN
+            target_devices: cuda:gpu
           - name: Intel / Arc A-Series Graphics
             runner: '["Linux", "arc"]'
             target_devices: level_zero:gpu;opencl:gpu;level_zero_v2:gpu         
@@ -248,6 +253,11 @@ jobs:
             runner: '["Linux", "cuda"]'
             image_options: -u 1001 --gpus all --cap-add SYS_ADMIN
             target_devices: cuda:gpu
+          - name: NVIDIA/CUDA 13.1
+            runner: '["Linux", "cuda13"]'
+            image: "ghcr.io/intel/llvm/ubuntu2404_intel_drivers_cuda131:latest"
+            image_options: -u 1001 --gpus all --cap-add SYS_ADMIN
+            target_devices: cuda:gpu
 
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     with:
@@ -288,6 +298,10 @@ jobs:
           - name: CUDA system
             runner: '["Linux", "cuda"]'
             image_extra_opts: --gpus all
+          - name: CUDA system with CUDA 13.1
+            runner: '["Linux", "cuda13"]'
+            image: "ghcr.io/intel/llvm/ubuntu2404_intel_drivers_cuda131:latest"
+            image_extra_opts: --gpus all
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     with:
       name: Perf tests on ${{ matrix.name }}

.github/workflows/sycl-linux-run-tests.yml

@@ -144,6 +144,14 @@ on:
         required: False
         default: ''
 
+      build_unified_runtime_locally:
+        description: |
+          If true, build unified-runtime locally using the container's CUDA
+          toolkit instead of using the prebuilt adapter from the toolchain.
+        type: string
+        required: False
+        default: 'true'
+
       in_workflow_call_mode:
         description: |
           Unlike `github.event_type` that captures the original trigger, we use
@@ -344,6 +352,7 @@ jobs:
         target_devices: ${{ inputs.target_devices }}
         extra_lit_opts: ${{ inputs.extra_lit_opts }}
         retention-days: ${{ inputs.retention-days }}
+        build_unified_runtime_locally: ${{ inputs.build_unified_runtime_locally }}
 
     - name: Run SYCL CTS Tests
       if: inputs.tests_selector == 'cts'

.github/workflows/ur-build-hw.yml

@@ -133,6 +133,7 @@ jobs:
         ${{ matrix.adapter.other_name != '' && format('-DUR_BUILD_ADAPTER_{0}=ON', matrix.adapter.other_name) || '' }}
         -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
         -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}
+        ${{ matrix.adapter.name == 'CUDA' && '-DUR_CONFORMANCE_NVIDIA_ARCH="sm_75"' || '' }}
         -DUR_DPCXX=./dpcpp_compiler/bin/clang++
         -DUR_SYCL_LIBRARY_DIR=./dpcpp_compiler/lib
         -DCMAKE_INSTALL_PREFIX=./install

.github/workflows/ur-precommit.yml

@@ -73,6 +73,14 @@ jobs:
           - name: L0_V2
             runner: UR_L0_BMG
             image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+
+            # 13.1
+          - name: CUDA
+            runner: cuda13
+            docker_image: "ghcr.io/intel/llvm/ubuntu2404_intel_drivers_cuda131:latest"
+            image_options: -u 1001 --privileged --cap-add SYS_ADMIN --gpus all
+
+          # 12.6.3
           - name: CUDA
             runner: UR_CUDA
             image_options: -u 1001 --privileged --cap-add SYS_ADMIN --gpus all

devops/actions/cleanup/cleanup.js

@@ -1,4 +1,4 @@
 const path = process.env.GITHUB_WORKSPACE + '/*';
 console.log('Cleaning ' + path)
-require('child_process').execSync('rm -rf ' + path);
-require('child_process').execSync('rm -rf ' + process.env.GITHUB_WORKSPACE + '/.git');
+// require('child_process').execSync('rm -rf ' + path);
+// require('child_process').execSync('rm -rf ' + process.env.GITHUB_WORKSPACE + '/.git');

devops/actions/run-tests/linux/e2e/action.yml

@@ -18,6 +18,9 @@
     required: false
   sycl_compiler:
     required: false
+  build_unified_runtime_locally:
+    required: false
+    default: 'true'
 
 
 runs:
@@ -52,6 +55,78 @@
       tar -I 'zstd' -xf /sycl-prebuilt/e2e_binaries.tar.zst -C build-e2e
       tar -I 'zstd' -xf /sycl-prebuilt/e2e_sources.tar.zst -C llvm
 
+  - name: Checkout unified-runtime source
+    if: inputs.build_unified_runtime_locally == 'true'
+    uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    with:
+      path: unified-runtime-src
+      sparse-checkout: |
+        unified-runtime/
+      ref: ${{ inputs.ref || github.sha }}
+
+  - name: Provide lit binary for UR tests
+    if: inputs.build_unified_runtime_locally == 'true'
+    shell: bash
+    run: |
+      if command -v lit >/dev/null 2>&1; then
+        exit 0
+      fi
+
+      mkdir -p "$HOME/.local/bin"
+
+      if [ -x "$PWD/toolchain/bin/llvm-lit" ]; then
+        ln -sfn "$PWD/toolchain/bin/llvm-lit" "$HOME/.local/bin/lit"
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+        exit 0
+      fi
+
+      if command -v llvm-lit >/dev/null 2>&1; then
+        ln -sfn "$(command -v llvm-lit)" "$HOME/.local/bin/lit"
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+        exit 0
+      fi
+
+      python3 -m pip install --user lit
+      echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+  - name: Build unified-runtime with container CUDA
+    if: inputs.build_unified_runtime_locally == 'true'
+    shell: bash
+    run: |
+      cd unified-runtime-src
+      ln -sfn "$PWD/../toolchain" dpcpp_compiler
+      cmake -S unified-runtime -B build \
+        -DCMAKE_C_COMPILER=gcc \
+        -DCMAKE_CXX_COMPILER=g++ \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DUR_ENABLE_TRACING=ON \
+        -DUR_DEVELOPER_MODE=ON \
+        -DUR_BUILD_TESTS=ON \
+        -DUR_BUILD_ADAPTER_L0_V2=ON \
+        -DUR_STATIC_LOADER=OFF \
+        -DUR_STATIC_ADAPTER_L0_V2=OFF \
+        -DUR_DPCXX=./dpcpp_compiler/bin/clang++ \
+        -DUR_SYCL_LIBRARY_DIR=./dpcpp_compiler/lib \
+        -DUR_CONFORMANCE_NVIDIA_ARCH="sm_75" \
+        -DCMAKE_INSTALL_PREFIX=./install
+      cmake --build build
+      cmake --install build
+
+  - name: Install built unified-runtime
+    if: inputs.build_unified_runtime_locally == 'true'
+    shell: bash
+    run: |
+      # Install the locally-built UR libraries over the prebuilt ones
+      mkdir -p "$PWD/toolchain/lib/plugins"
+      cp unified-runtime-src/install/lib/libur_adapter_cuda.so* "$PWD/toolchain/lib/" 2>/dev/null || true
+      cp unified-runtime-src/install/lib/libur_loader.so* "$PWD/toolchain/lib/" 2>/dev/null || true
+      cp unified-runtime-src/install/lib/plugins/* "$PWD/toolchain/lib/plugins/" 2>/dev/null || true
+      cp unified-runtime-src/build/lib/libur_adapter_cuda.so* "$PWD/toolchain/lib/" 2>/dev/null || true
+      cp unified-runtime-src/build/lib/libur_loader.so* "$PWD/toolchain/lib/" 2>/dev/null || true
+      cp unified-runtime-src/build/lib/plugins/* "$PWD/toolchain/lib/plugins/" 2>/dev/null || true
+      # Also ensure the built libs are in LD_LIBRARY_PATH
+      echo LD_LIBRARY_PATH="$PWD/unified-runtime-src/install/lib:$PWD/unified-runtime-src/build/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+
   - name: Deduce E2E CMake options
     if: inputs.testing_mode != 'run-only'
     id: cmake_opts
@@ -110,4 +185,4 @@
     if: ${{ !cancelled() }}
     shell: bash
     run: |
-      rm -rf build-e2e
+      # rm -rf build-e2e

devops/containers/ubuntu2404_build_cuda131.Dockerfile

@@ -0,0 +1,71 @@
+FROM nvidia/cuda:13.1.0-devel-ubuntu24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+USER root
+
+# Configure LLVM nightly repo
+RUN apt-get update -qq && apt-get install --no-install-recommends -yqq curl ca-certificates
+RUN curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key -o /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+RUN echo 'deb http://apt.llvm.org/noble/ llvm-toolchain-noble main' > /etc/apt/sources.list.d/llvm.list
+RUN echo 'deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main' > /etc/apt/sources.list.d/llvm.list
+
+# Install SYCL prerequisites
+COPY scripts/install_build_tools.sh /install.sh
+RUN /install.sh
+
+COPY scripts/install_vulkan.sh /install_vulkan.sh
+RUN /install_vulkan.sh
+
+# libzstd-dev installed by default on Ubuntu 24.04 is not compiled with -fPIC flag.
+# This causes linking errors when building SYCL runtime.
+# Bug: https://github.com/intel/llvm/issues/15935
+# Workaround: build zstd from sources with -fPIC flag.
+COPY scripts/build_zstd.sh /build_zstd.sh
+RUN /build_zstd.sh
+
+SHELL ["/bin/bash", "-ec"]
+
+# Make the directory if it doesn't exist yet.
+# This location is recommended by the distribution maintainers.
+RUN mkdir --parents --mode=0755 /etc/apt/keyrings
+# Download the key, convert the signing-key to a full
+# keyring required by apt and store in the keyring directory
+RUN curl -sSL https://repo.radeon.com/rocm/rocm.gpg.key | \
+gpg --dearmor | tee /etc/apt/keyrings/rocm.gpg > /dev/null && \
+# Add rocm repo
+echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.3/ubuntu noble main" \
+    | tee /etc/apt/sources.list.d/amdgpu.list && \
+echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.3 noble main" \
+    |  tee --append /etc/apt/sources.list.d/rocm.list && \
+echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' \
+    | tee /etc/apt/preferences.d/rocm-pin-600 && \
+echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' \
+    | tee /etc/apt/preferences.d/rocm-pin-600
+# Install the ROCM kernel driver
+RUN apt update && apt install -yqq rocm-dev && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+# Fix Vulkan install inside container
+# https://stackoverflow.com/questions/74965945/vulkan-is-unable-to-detect-nvidia-gpu-from-within-a-docker-container-when-using
+RUN apt-get update && \
+    apt-get install -y libegl1 && \
+    apt-get install -y --no-install-recommends --download-only libnvidia-gl-565 && \
+    dpkg-deb --extract /var/cache/apt/archives/libnvidia-gl-565_*.deb extracted && \
+    cp -R ./extracted/usr/* /usr/ && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*.deb ./extracted
+
+COPY scripts/create-sycl-user.sh /user-setup.sh
+RUN /user-setup.sh
+
+COPY scripts/docker_entrypoint.sh /docker_entrypoint.sh
+
+COPY actions/cached_checkout /actions/cached_checkout
+COPY actions/cleanup /actions/cleanup
+COPY scripts/install_drivers.sh /opt/install_drivers.sh
+
+USER sycl
+
+ENTRYPOINT ["/docker_entrypoint.sh"]
+

devops/containers/ubuntu2404_intel_drivers_cuda131.Dockerfile

@@ -0,0 +1,25 @@
+ARG base_tag=latest
+ARG base_image=ghcr.io/intel/llvm/ubuntu2404_build_cuda131
+
+FROM $base_image:$base_tag
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+USER root
+
+RUN apt update && apt install -yqq wget
+
+COPY scripts/install_drivers.sh /
+COPY dependencies.json /
+
+RUN mkdir /runtimes
+ENV INSTALL_LOCATION=/runtimes
+RUN --mount=type=secret,id=github_token \
+    GITHUB_TOKEN=$(cat /run/secrets/github_token) /install_drivers.sh dependencies.json --all
+
+COPY scripts/drivers_entrypoint.sh /drivers_entrypoint.sh
+
+USER sycl
+
+ENTRYPOINT ["/bin/bash", "/drivers_entrypoint.sh"]
+