build-wheels-cuda-windows.yml

# From https://github.com/pytorch/test-infra/wiki/Using-Nova-Reusable-Build-Workflows
name: Build Windows CUDA Wheels

on:
  pull_request:
    paths:
      - .ci/**/*
      - .github/scripts/filter_cuda_matrix.py
      - .github/workflows/build-wheels-cuda-windows.yml
      - '**/CMakeLists.txt'
      - backends/cuda/**/*
      - extension/cuda/**/*
      - install_utils.py
      - pyproject.toml
      - setup.py
      - docs/source/using-executorch-cpp.md
      - tools/cmake/**/*
      # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories
      # setup.py copies from, so adding one there cannot silently drop it from this list.
      - devtools/etdump/**.h
      - extension/**.h
      - runtime/**.h
  push:
    branches:
      - nightly
      - release/*
    tags:
      # NOTE: Binary build pipelines should only get triggered on release candidate builds
      # Release candidate tags look like: v1.11.0-rc1
      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
      - ciflow/binaries/*
  workflow_dispatch:

permissions:
  id-token: write
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
  cancel-in-progress: true

jobs:
  generate-matrix:
    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
    with:
      package-type: wheel
      os: windows
      test-infra-repository: pytorch/test-infra
      test-infra-ref: main
      with-cuda: enable
      with-cpu: disable
      with-rocm: disable
      python-versions: '["3.11", "3.12", "3.13", "3.14", "3.15"]'

  # The same filter the Linux CUDA rows use: only the trains this project supports, and one row
  # on a pull request. It fails rather than emitting an empty matrix, because a workflow with no
  # build job reads as a pass.
  filter-matrix:
    needs: generate-matrix
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.filter.outputs.matrix }}
    steps:
      - uses: actions/setup-python@v6
        with:
          python-version: '3.12'
      - uses: actions/checkout@v4
      - name: Filter the matrix
        id: filter
        run: |
          set -eou pipefail
          MATRIX_BLOB=${{ toJSON(needs.generate-matrix.outputs.matrix) }}
          LIMIT_PR=${{ (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ciflow/binaries/all')) && 'true' || 'false' }}
          MATRIX_BLOB="$(python3 .github/scripts/filter_cuda_matrix.py \
            --matrix "${MATRIX_BLOB}" --limit-pr-builds "${LIMIT_PR}")"
          echo "${MATRIX_BLOB}"
          echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}"

  build:
    needs: filter-matrix
    strategy:
      fail-fast: false
      matrix:
        include:
          - repository: pytorch/executorch
            pre-script: .ci\\scripts\\wheel\\pre_build_script.sh
            env-script: .ci\\scripts\\wheel\\vc_env_helper.bat
            post-script: .ci\\scripts\\wheel\\post_build_script.sh
            smoke-test-script: .ci/scripts/wheel/test_cuda_windows.py
            package-name: executorch
    name: ${{ matrix.repository }}
    uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main
    with:
      repository: ${{ matrix.repository }}
      ref: ""
      test-infra-repository: pytorch/test-infra
      # Avoid the unsatisfiable conda upgrade in the Windows runner's base env.
      # https://github.com/pytorch/test-infra/pull/9179
      test-infra-ref: e3d344d886a7281f6a919115d332df2a080da8ad
      build-matrix: ${{ needs.filter-matrix.outputs.matrix }}
      pre-script: ${{ matrix.pre-script }}
      env-script: ${{ matrix.env-script }}
      post-script: ${{ matrix.post-script }}
      package-name: ${{ matrix.package-name }}
      smoke-test-script: ${{ matrix.smoke-test-script }}
      trigger-event: ${{ github.event_name }}
      wheel-build-params: "--verbose"
      # The device code for every GPU the row serves takes most of the default hour to
      # build, which left the smoke test cut off and no wheel uploaded.
      timeout: 150
      # Submodules are initialized in pre_build_script.sh with OpenSSL to avoid
      # schannel SSL errors on Windows when cloning from non-GitHub hosts.
      submodules: false

  docker-image:
    name: Resolve CI docker image
    uses: ./.github/workflows/_docker-image.yml

  # The trains the end-to-end jobs cover: exactly the ones published as wheels, read from the
  # list filter_cuda_matrix.py builds the release rows from, so adding or dropping a train there
  # changes what is tested here too.
  cuda-trains:
    needs: filter-matrix
    runs-on: ubuntu-latest
    outputs:
      versions: ${{ steps.list.outputs.versions }}
      trains: ${{ steps.list.outputs.trains }}
    steps:
      - id: list
        env:
          MATRIX: ${{ needs.filter-matrix.outputs.matrix }}
        run: |
          set -eou pipefail
          # The trains the build job actually builds, from its own filtered matrix, so the
          # end-to-end jobs follow it exactly: a pull request without the
          # ciflow/binaries/all label builds one train, and a train the generator dropped
          # is not tested against a wheel that does not exist. Per train, the oldest Python
          # it was built for, whose published wheel the end-to-end job installs.
          python3 - <<'EOF' >> "${GITHUB_OUTPUT}"
          import json, os
          rows = json.loads(os.environ["MATRIX"])["include"]
          trains = {}
          for row in rows:
              version, python = row["gpu_arch_version"], row["python_version"]
              key = tuple(int(p) for p in python.split("."))
              if version not in trains or key < trains[version][0]:
                  trains[version] = (key, python, row["desired_cuda"])
          entries = [
              {"cuda-version": v, "python": p, "cu": cu}
              for v, (_, p, cu) in sorted(trains.items())
          ]
          print("versions=" + json.dumps([e["cuda-version"] for e in entries]))
          print("trains=" + json.dumps(entries))
          EOF
          cat "${GITHUB_OUTPUT}"

  # The route a Windows user has, per CUDA train: lowering for CUDA cannot run on Windows,
  # so a Linux job exports a program for a Windows target with that train's torch, and a
  # Windows GPU job builds the wheel with that train's nvcc and runs the program through
  # the wheel's C++ SDK. The build job above has no GPU and no program, so this is what
  # executes CUDA code. Modeled on cuda-windows.yml, which does the same for models.
  export-cuda-windows-wheel-program:
    needs: [docker-image, cuda-trains]
    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request'
    strategy:
      fail-fast: false
      matrix:
        cuda-version: ${{ fromJSON(needs.cuda-trains.outputs.versions) }}
    name: export-cuda-windows-wheel-program-${{ matrix.cuda-version }}
    uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
    permissions:
      id-token: write
      contents: read
    with:
      timeout: 90
      runner: mt-l-x86aavx2-11-41-a10g
      gpu-arch-type: cuda
      gpu-arch-version: ${{ matrix.cuda-version }}
      # The image carries the MinGW cross compiler; each train's CUDA comes from the redist
      # archives below.
      docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-cuda-windows-${{ needs.docker-image.outputs.ci-docker-hash }}
      submodules: recursive
      upload-artifact: cuda-windows-wheel-program-${{ matrix.cuda-version }}
      # The commit the build job's empty ref checks out (on a pull request, its merge commit),
      # so the wheel, the program and the scripts that test them come from one commit.
      ref: ${{ github.sha }}
      script: |
        set -eux
        TRAIN="${{ matrix.cuda-version }}"
        # The pybindings need GLIBCXX_3.4.30, which the image's conda libstdc++ lacks.
        mv /opt/conda/lib/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6.bak || true
        ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6
        # Each train gets its own Linux nvcc, which decides the torch that
        # install_executorch picks, and its own Windows runtime to link against.
        # Under the job's temp directory: the image runs as ci-user, which cannot
        # write to /opt.
        TOOLKITS="${RUNNER_TEMP:-/tmp}/cuda-toolkits"
        mkdir -p "${TOOLKITS}" 2>/dev/null || TOOLKITS=/tmp/cuda-toolkits
        mkdir -p "${TOOLKITS}"
        python .ci/scripts/wheel/install_cuda_redist.py --train "${TRAIN}" \
          --platform linux-x86_64 --dest "${TOOLKITS}/cuda-${TRAIN}"
        export CUDA_HOME="${TOOLKITS}/cuda-${TRAIN}"
        export PATH="${CUDA_HOME}/bin:${PATH}"
        export CUDACXX="${CUDA_HOME}/bin/nvcc"
        python .ci/scripts/wheel/install_cuda_redist.py --train "${TRAIN}" \
          --platform windows-x86_64 --dest "${TOOLKITS}/cuda-windows-${TRAIN}" --components cuda_cudart
        export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${TRAIN}"
        x86_64-w64-mingw32-g++ --version
        nvcc --version
        ls -la "${WINDOWS_CUDA_HOME}/bin/x64"
        export USE_MKL=OFF
        PYTHON_EXECUTABLE=python ./install_executorch.sh
        python -c "import torch; v = torch.version.cuda; assert v == '${TRAIN}', v"
        python .ci/scripts/wheel/test_cuda_windows.py --export "${RUNNER_ARTIFACT_DIR}"

  # The published wheel and the program it runs, in one artifact: windows_job.yml downloads
  # a single artifact and gives its script no token to fetch another.
  bundle-cuda-windows-wheel-e2e:
    needs: [cuda-trains, build, export-cuda-windows-wheel-program]
    # Per train, like the end-to-end job: a train whose wheel or program is missing fails
    # here on the download, without skipping the others.
    if: ${{ !cancelled() && needs.cuda-trains.result == 'success' && (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.cuda-trains.outputs.trains) }}
    name: bundle-cuda-windows-wheel-e2e-${{ matrix.cuda-version }}
    runs-on: ubuntu-latest
    steps:
      - name: Download the exported program
        uses: actions/download-artifact@v4
        with:
          name: cuda-windows-wheel-program-${{ matrix.cuda-version }}
          path: bundle
      # The name build_wheels_windows.yml uploads under; the build job passes no ref.
      - name: Download the published wheel
        uses: actions/download-artifact@v4
        with:
          name: pytorch_executorch__${{ matrix.python }}_${{ matrix.cu }}_x64
          path: bundle
      - run: ls -la bundle
      - uses: actions/upload-artifact@v4
        with:
          name: cuda-windows-wheel-e2e-${{ matrix.cuda-version }}
          path: bundle/
          if-no-files-found: error
          retention-days: 3

  test-cuda-windows-wheel-e2e:
    needs: [cuda-trains, bundle-cuda-windows-wheel-e2e]
    # Not gated on the bundle job's result: that is one result for the whole matrix, so a
    # single train failing skipped every train here. Each train downloads its own bundle,
    # and a train whose bundle is missing fails here on the download. The fork condition is
    # the export job's own: !cancelled() would otherwise run this on a fork pull request,
    # where nothing was exported, and fail on the missing artifact.
    if: ${{ !cancelled() && needs.cuda-trains.result == 'success' && (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') }}
    strategy:
      fail-fast: false
      matrix:
        cuda-version: ${{ fromJSON(needs.cuda-trains.outputs.versions) }}
    name: test-cuda-windows-wheel-e2e-${{ matrix.cuda-version }}
    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
    with:
      timeout: 120
      runner: windows.g5.4xlarge.nvidia.gpu
      gpu-arch-type: cuda
      gpu-arch-version: ${{ matrix.cuda-version }}
      download-artifact: cuda-windows-wheel-e2e-${{ matrix.cuda-version }}
      # The commit the build job's empty ref checks out (on a pull request, its merge commit),
      # so the wheel, the program and the scripts that test them come from one commit.
      ref: ${{ github.sha }}
      script: |
        git config --global http.sslBackend openssl
        git submodule update --init --recursive
        conda init powershell
        powershell -Command "& {
          Set-PSDebug -Trace 1
          \$ErrorActionPreference = 'Stop'
          \$PSNativeCommandUseErrorActionPreference = \$true
          .ci/scripts/wheel/test_cuda_windows_e2e.ps1 -Train '${{ matrix.cuda-version }}' -Artifacts \$env:RUNNER_ARTIFACT_DIR
        }"