From d84d4d5258cf04ca38fc0c17c39af1015aef5165 Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Wed, 10 Sep 2025 14:44:51 +0000 Subject: [PATCH 01/11] Add torchcodec inspired linux gpu CI workflow --- .github/workflows/unittest-linux-gpu.yml | 275 +++++++++++++++-------- 1 file changed, 182 insertions(+), 93 deletions(-) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index 3d785db097..81d5895a6e 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -1,98 +1,187 @@ name: Unit-tests on Linux GPU on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: + pull_request: + push: + branches: + - main + workflow_dispatch: jobs: - tests: - strategy: - matrix: - # TODO add up to 3.13 - python_version: ["3.10"] - cuda_arch_version: ["12.6"] - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - with: - runner: linux.g5.4xlarge.nvidia.gpu - repository: pytorch/audio - gpu-arch-type: cuda - gpu-arch-version: ${{ matrix.cuda_arch_version }} - timeout: 120 - - script: | - set -ex - # Set up Environment Variables - export PYTHON_VERSION="${{ matrix.python_version }}" - export PIP_PROGRESS_BAR=off - export CONDA_QUIET=1 - export CU_VERSION="${{ matrix.cuda_arch_version }}" - export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}" - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true - # Avoid reproducibility errors with CUBLAS: https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility - export CUBLAS_WORKSPACE_CONFIG=:4096:8 - - # Set UPLOAD_CHANNEL - if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export UPLOAD_CHANNEL=test - else - export UPLOAD_CHANNEL=nightly - fi - - echo "::group::Create conda env" - # Mark Build Directory Safe - git config --global --add safe.directory /__w/audio/audio - conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}" - conda activate ./ci_env - - echo "::endgroup::" - echo "::group::Install PyTorch" - export GPU_ARCH_ID="cu126" # TODO this is currently hardcoded, should depend on matrix's value. - PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${GPU_ARCH_ID}" - pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}" - echo "::endgroup::" - - echo "::group::Install TorchAudio" - conda install --quiet --yes cmake ninja - pip3 install --progress-bar off -v -e . --no-use-pep517 - - echo "::endgroup::" - echo "::group::Build FFmpeg" - .github/scripts/ffmpeg/build_gpu.sh - - echo "::endgroup::" - echo "::group::Install other Dependencies" - - pip3 install parameterized requests coverage pytest pytest-cov scipy numpy expecttest - - echo "::endgroup::" - echo "::group::Run tests" - export PATH="${PWD}/third_party/install/bin/:${PATH}" - - declare -a args=( - '-v' - '--cov=torchaudio' - "--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml" - '--durations' '100' - '-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency' - ) - - cd test - python3 -m torch.utils.collect_env - env | grep TORCHAUDIO || true - pytest "${args[@]}" torchaudio_unittest - coverage html + build: + # Do not use matrix here to parameterize Python/CUDA versions. + # This job is required to pass for each PR. + # The name of the required job is sensitive to matrix parameter. + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + with: + job-name: Build doc + runner: linux.g5.4xlarge.nvidia.gpu + repository: pytorch/audio + gpu-arch-type: cuda + gpu-arch-version: "12.6" # See GPU_ARCH_ID below + timeout: 120 + + script: | + set -ex + + # Set up Environment Variables + export PYTHON_VERSION="3.10" + export CU_VERSION="11.8" + export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}" + export PIP_PROGRESS_BAR=off + export CONDA_QUIET=1 + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true + # Avoid reproducibility errors with CUBLAS: https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility + export CUBLAS_WORKSPACE_CONFIG=:4096:8 + + # Set CHANNEL + if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + export CHANNEL=test + export BUILD_VERSION="$( cut -f 1 -d a version.txt )" + else + export CHANNEL=nightly + export BUILD_VERSION="$( cut -f 1 -d a version.txt )".dev"$(date "+%Y%m%d")" + fi + + # Install miniforge + wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" + bash Miniforge3.sh -b -p "${HOME}/conda" + source "${HOME}/conda/etc/profile.d/conda.sh" + + echo "::group::Create conda env" + # Mark Build Directory Safe + git config --global --add safe.directory /__w/audio/audio + conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}" + conda activate ./ci_env + conda install -q -y pip numpy + + echo "::endgroup::" + echo "::group::Install PyTorch" + + GPU_ARCH_ID=cu126 # This is hard-coded and must be consistent with gpu-arch-version. + PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL}/${GPU_ARCH_ID}" + pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}" + + echo "::endgroup::" + echo "::group::Install TorchAudio" + conda install --quiet --yes cmake>=3.18.0 ninja + pip install --progress-bar off -v . --no-build-isolation + + # TODO: Need to rely on torchcodec instead of building ffmpeg from source. + echo "::endgroup::" + echo "::group::Build FFmpeg" + conda install -q -y "ffmpeg<=7" + + echo "::endgroup::" + echo "::group::Install other dependencies" + + pip install --progress-bar off parameterized requests coverage pytest pytest-cov scipy numpy expecttest + + declare -a args=( + '-v' + '--cov=torchaudio' + "--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml" + '--durations' '100' + '-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency' + ) + + cd test + python3 -m torch.utils.collect_env + env | grep TORCHAUDIO || true + pytest "${args[@]}" torchaudio_unittest + coverage html + + commit-main: + if: github.ref_name == 'main' + permissions: + # Required for `git push` + # Note: + # This is not effective from fork. + # When you debug this, make sure to make a branch on pytorch and + # make PR from there. + contents: write + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v3 + with: + ref: gh-pages + fetch-depth: 5 + - uses: actions/download-artifact@v3 + with: + name: docs + - name: Update main doc + run: | + set -x + + git config user.name "pytorchbot" + git config user.email "soumith+bot@pytorch.org" + # When `git clone`, `gh-pages` branch is fetched by default. + # The size of gh-pages grows significantly, so we use ammend and force push + # We add a new commit once a week + if [ "$(date +%d)" = "1" ]; then + git commit --allow-empty -m "placeholder" + fi + + # TODO: add tag-based process (need to handle the main directory name) + # Update the main doc + rm -rf main + mv html main + git add --all main || true + + git commit --amend -m "auto-generating sphinx docs" || true + git push -f + + # Push for release + # Make sure that version.txt is updated first (alpha suffix is removed) + commit-release: + if: startsWith(github.ref_name, 'release/') && ( github.event_name == 'workflow_dispatch' ) + permissions: + # Required for `git push` + # Note: + # This is not effective from fork. + # When you debug this, make sure to make a branch on pytorch and + # make PR from there. + contents: write + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v3 + with: + ref: gh-pages + fetch-depth: 5 + - uses: actions/checkout@v4 + with: + path: _src + - uses: actions/download-artifact@v3 + with: + name: docs + - name: Update doc + run: | + set -x + + git config user.name "pytorchbot" + git config user.email "soumith+bot@pytorch.org" + # When `git clone`, `gh-pages` branch is fetched by default. + # The size of gh-pages grows significantly, so we use ammend and force push + # We add a new commit once a week + if [ "$(date +%d)" = "1" ]; then + git commit --allow-empty -m "placeholder" + fi + + dirname="$(cat _src/version.txt)" + rm -rf "${dirname}" + mv html "${dirname}" + git add --all "${dirname}" || true + + git commit --amend -m "auto-generating sphinx docs" || true + git push -f From a425a0d21af569f76034b0e7590c69a3b96db303 Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Wed, 10 Sep 2025 15:47:17 +0000 Subject: [PATCH 02/11] Add back nightly and release triggers --- .github/workflows/unittest-linux-gpu.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index 81d5895a6e..be38806434 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -4,7 +4,9 @@ on: pull_request: push: branches: + - nightly - main + - release/* workflow_dispatch: jobs: From 820ceade75da341147402b4e8b29521e0e96579b Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Wed, 10 Sep 2025 17:52:27 +0000 Subject: [PATCH 03/11] Change job name --- .github/workflows/unittest-linux-gpu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index be38806434..c8d6e331a7 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: jobs: - build: + tests: # Do not use matrix here to parameterize Python/CUDA versions. # This job is required to pass for each PR. # The name of the required job is sensitive to matrix parameter. @@ -19,7 +19,7 @@ jobs: id-token: write contents: read with: - job-name: Build doc + job-name: Run tests runner: linux.g5.4xlarge.nvidia.gpu repository: pytorch/audio gpu-arch-type: cuda From 84400db5fc59cce52e2fdeb9230ad6ba8c8f1725 Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Wed, 10 Sep 2025 17:54:32 +0000 Subject: [PATCH 04/11] Change 'tests' back to 'build' in CI file --- .github/workflows/unittest-linux-gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index c8d6e331a7..7d6ca0e477 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: jobs: - tests: + build: # Do not use matrix here to parameterize Python/CUDA versions. # This job is required to pass for each PR. # The name of the required job is sensitive to matrix parameter. From c83d1f7fa7bd208d3e8453ac2f80ff0eec045d11 Mon Sep 17 00:00:00 2001 From: Sam Anklesaria <61420+samanklesaria@users.noreply.github.com> Date: Thu, 11 Sep 2025 03:01:50 -0500 Subject: [PATCH 05/11] Remove ffmpeg use in github workflows (#4083) --- .github/scripts/ffmpeg/build.bat | 9 -- .github/scripts/ffmpeg/build.sh | 163 ---------------------------- .github/scripts/ffmpeg/build_gpu.sh | 62 ----------- .github/workflows/ffmpeg.yml | 108 ------------------ 4 files changed, 342 deletions(-) delete mode 100644 .github/scripts/ffmpeg/build.bat delete mode 100755 .github/scripts/ffmpeg/build.sh delete mode 100755 .github/scripts/ffmpeg/build_gpu.sh delete mode 100644 .github/workflows/ffmpeg.yml diff --git a/.github/scripts/ffmpeg/build.bat b/.github/scripts/ffmpeg/build.bat deleted file mode 100644 index 0ab0d8eef2..0000000000 --- a/.github/scripts/ffmpeg/build.bat +++ /dev/null @@ -1,9 +0,0 @@ -@echo off - -set PROJ_FOLDER=%cd% - -choco install -y --no-progress msys2 --package-parameters "/NoUpdate" -C:\tools\msys64\usr\bin\env MSYSTEM=MINGW64 /bin/bash -l -c "pacman -S --noconfirm --needed base-devel mingw-w64-x86_64-toolchain diffutils" -C:\tools\msys64\usr\bin\env MSYSTEM=MINGW64 /bin/bash -l -c "cd ${PROJ_FOLDER} && packaging/vc_env_helper.bat bash .github/scripts/ffmpeg/build.sh" - -:end diff --git a/.github/scripts/ffmpeg/build.sh b/.github/scripts/ffmpeg/build.sh deleted file mode 100755 index 03cfc478d8..0000000000 --- a/.github/scripts/ffmpeg/build.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# This script builds FFmpeg libraries without any functional features. -# -# IMPORTANT: -# The resulting library files have to be LGPL version of FFmpeg libraries. -# - Do not enable `--enable-nonfree` and `--enable-gpl`. -# - Do not enable third party library integrations like x264. -# -# This script is not meant to build useful FFmpeg libraries, but to build -# a skeleton of FFmpeg libraries that are use only during the build process of -# torchaudio. -# -# The resulting FFmpeg libraries should not be shipped either. - -set -eux - -prefix="${FFMPEG_ROOT}" -args="" -if [[ "$OSTYPE" == "msys" ]]; then - args="--toolchain=msvc" -fi -ffmpeg_version="${FFMPEG_VERSION:-4.1.8}" - -if [[ "${ffmpeg_version}" == "master" ]]; then - archive="https://github.com/FFmpeg/FFmpeg/archive/master.tar.gz" -else - archive="https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n${ffmpeg_version}.tar.gz" -fi - -build_dir=$(mktemp -d -t ffmpeg-build.XXXXXXXXXX) -cleanup() { - rm -rf "${build_dir}" -} -trap 'cleanup $?' EXIT - -( -cd "${build_dir}" -# NOTE: -# When changing the version of FFmpeg, update the README so that the link to the source points -# the same version. -curl -LsS -o ffmpeg.tar.gz "${archive}" -tar -xf ffmpeg.tar.gz --strip-components 1 -./configure \ - --prefix="${prefix}" \ - --disable-all \ - --disable-everything \ - --disable-programs \ - --disable-doc \ - --disable-debug \ - --disable-autodetect \ - --disable-x86asm \ - --disable-iconv \ - --disable-encoders \ - --disable-decoders \ - --disable-hwaccels \ - --disable-muxers \ - --disable-demuxers \ - --disable-parsers \ - --disable-bsfs \ - --disable-protocols \ - --disable-devices \ - --disable-filters \ - --disable-asm \ - --disable-static \ - --enable-shared \ - --enable-rpath \ - --enable-pic \ - --enable-avcodec \ - --enable-avdevice \ - --enable-avfilter \ - --enable-avformat \ - --enable-avutil ${args} - -make -j install -ls ${prefix}/* -) - -# macOS: Fix rpath so that the libraries are searched dynamically in user environment. -# In Linux, this is handled by `--enable-rpath` flag. -if [[ "$(uname)" == Darwin ]]; then - major_ver=${ffmpeg_version:0:1} - if [[ ${major_ver} == 4 ]]; then - avutil=libavutil.56 - avcodec=libavcodec.58 - avformat=libavformat.58 - avdevice=libavdevice.58 - avfilter=libavfilter.7 - elif [[ ${major_ver} == 5 ]]; then - avutil=libavutil.57 - avcodec=libavcodec.59 - avformat=libavformat.59 - avdevice=libavdevice.59 - avfilter=libavfilter.8 - elif [[ ${ffmpeg_version} == master || ${major_ver} == 6 ]]; then - avutil=libavutil.58 - avcodec=libavcodec.60 - avformat=libavformat.60 - avdevice=libavdevice.60 - avfilter=libavfilter.9 - else - printf "Error: unexpected FFmpeg major version: %s\n" ${major_ver} - exit 1; - fi - - otool="/usr/bin/otool" - # NOTE: miniconda has a version of otool and install_name_tool installed and we want - # to use the default sytem version instead of the miniconda version since the miniconda - # version can produce inconsistent results - - # Attempt to use /usr/bin/otool as our default otool - if [[ ! -e ${otool} ]]; then - otool="$(which otool)" - fi - install_name_tool="/usr/bin/install_name_tool" - # Attempt to use /usr/bin/install_name_tool as our default install_name_tool - if [[ ! -e ${install_name_tool} ]]; then - install_name_tool="$(which install_name_tool)" - fi - - # list up the paths to fix - for lib in ${avcodec} ${avdevice} ${avfilter} ${avformat} ${avutil}; do - ${otool} -l ${prefix}/lib/${lib}.dylib | grep -B2 ${prefix} - done - - # Replace the hardcoded paths to @rpath - ${install_name_tool} \ - -change ${prefix}/lib/${avutil}.dylib @rpath/${avutil}.dylib \ - -delete_rpath ${prefix}/lib \ - -id @rpath/${avcodec}.dylib \ - ${prefix}/lib/${avcodec}.dylib - ${otool} -l ${prefix}/lib/${avcodec}.dylib | grep -B2 ${prefix} - - ${install_name_tool} \ - -change ${prefix}/lib/${avformat}.dylib @rpath/${avformat}.dylib \ - -change ${prefix}/lib/${avcodec}.dylib @rpath/${avcodec}.dylib \ - -change ${prefix}/lib/${avutil}.dylib @rpath/${avutil}.dylib \ - -delete_rpath ${prefix}/lib \ - -id @rpath/${avdevice}.dylib \ - ${prefix}/lib/${avdevice}.dylib - ${otool} -l ${prefix}/lib/${avdevice}.dylib | grep -B2 ${prefix} - - ${install_name_tool} \ - -change ${prefix}/lib/${avutil}.dylib @rpath/${avutil}.dylib \ - -delete_rpath ${prefix}/lib \ - -id @rpath/${avfilter}.dylib \ - ${prefix}/lib/${avfilter}.dylib - ${otool} -l ${prefix}/lib/${avfilter}.dylib | grep -B2 ${prefix} - - ${install_name_tool} \ - -change ${prefix}/lib/${avcodec}.dylib @rpath/${avcodec}.dylib \ - -change ${prefix}/lib/${avutil}.dylib @rpath/${avutil}.dylib \ - -delete_rpath ${prefix}/lib \ - -id @rpath/${avformat}.dylib \ - ${prefix}/lib/${avformat}.dylib - ${otool} -l ${prefix}/lib/${avformat}.dylib | grep -B2 ${prefix} - - ${install_name_tool} \ - -delete_rpath ${prefix}/lib \ - -id @rpath/${avutil}.dylib \ - ${prefix}/lib/${avutil}.dylib - ${otool} -l ${prefix}/lib/${avutil}.dylib | grep -B2 ${prefix} -fi diff --git a/.github/scripts/ffmpeg/build_gpu.sh b/.github/scripts/ffmpeg/build_gpu.sh deleted file mode 100755 index 7cdeb7917a..0000000000 --- a/.github/scripts/ffmpeg/build_gpu.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# This script builds FFmpeg with NVIDIA Video codec SDK support. -# -# IMPORTANT: -# The resulting library files are non-distributable. -# Do not ship them. Do not use them for binary build. - -set -eux - -revision="${FFMPEG_VERSION:-6.0}" -codec_header_version="${CODEC_HEADER_VERSION:-n12.0.16.0}" -prefix="${PREFIX:-${CONDA_PREFIX}}" -ccap="${COMPUTE_CAPABILITY:-86}" - -# Build FFmpeg with NVIDIA Video Codec SDK -# TODO cache this -( - git clone --quiet https://git.videolan.org/git/ffmpeg/nv-codec-headers.git - cd nv-codec-headers - git checkout ${codec_header_version} - make PREFIX=${prefix} install -) - -conda install \ - --quiet --yes \ - -c conda-forge \ - yasm x264 gnutls pkg-config lame libopus libvpx openh264 openssl x264 - -( - wget -q https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n${revision}.tar.gz - tar -xf n${revision}.tar.gz - cd ./FFmpeg-n${revision} - # Sometimes, the resulting FFmpeg binaries have development version number. - # Setting `revision` variable ensures that it has the right version number. - export revision=${revision} - ./configure \ - --prefix="${prefix}" \ - --extra-cflags="-I${prefix}/include" \ - --extra-ldflags="-L${prefix}/lib" \ - --nvccflags="-gencode arch=compute_${ccap},code=sm_${ccap} -O2" \ - --disable-doc \ - --enable-rpath \ - --disable-static \ - --enable-protocol=https \ - --enable-gnutls \ - --enable-shared \ - --enable-gpl \ - --enable-nonfree \ - --enable-libmp3lame \ - --enable-libx264 \ - --enable-cuda-nvcc \ - --enable-nvenc \ - --enable-cuvid \ - --enable-nvdec - make clean - make -j > /dev/null 2>&1 - make install - # test - # src="https://download.pytorch.org/torchaudio/tutorial-assets/stream-api/NASAs_Most_Scientifically_Complex_Space_Observatory_Requires_Precision-MP4_small.mp4" - # ffmpeg -y -vsync 0 -hwaccel cuvid -hwaccel_output_format cuda -c:v h264_cuvid -resize 360x240 -i "${src}" -c:a copy -c:v h264_nvenc -b:v 5M test.mp4 -) diff --git a/.github/workflows/ffmpeg.yml b/.github/workflows/ffmpeg.yml deleted file mode 100644 index 1d589c07f7..0000000000 --- a/.github/workflows/ffmpeg.yml +++ /dev/null @@ -1,108 +0,0 @@ -# This job is not directly related to regular CI pipeline. -# It is intended to create FFmpeg binaries that we upload on S3, -# which then will be used during all the build process in CI or local. -# -# This job does not include uploading part. -# Upload needs to be done manually, and it should be done only once -# par new major release of FFmepg. -name: FFmpeg Binaries - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * 0' # on sunday - -jobs: - LGPL-Linux-x86_64: - strategy: - fail-fast: false - matrix: - ffmpeg_version: ["4.4.4", "5.1.4", "6.1.1", "master"] - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - with: - job-name: Build - upload-artifact: ffmpeg-lgpl - repository: pytorch/audio - script: | - export FFMPEG_VERSION="${{ matrix.ffmpeg_version }}" - export FFMPEG_ROOT="${PWD}/ffmpeg" - .github/scripts/ffmpeg/build.sh - - tar -cf ffmpeg.tar.gz ffmpeg/include ffmpeg/lib - - artifact_dir="${RUNNER_ARTIFACT_DIR}/$(date +%Y-%m-%d)/linux_x86_64/" - mkdir -p "${artifact_dir}" - mv ffmpeg.tar.gz "${artifact_dir}/${FFMPEG_VERSION}.tar.gz" - - LGPL-Linux-aarch64: - strategy: - fail-fast: false - matrix: - ffmpeg_version: ["4.4.4", "5.1.4", "6.1.1", "master"] - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - with: - job-name: Build - upload-artifact: ffmpeg-lgpl - repository: pytorch/audio - runner: linux.arm64.2xlarge - docker-image: pytorch/manylinuxaarch64-builder:cpu-aarch64 - script: | - export FFMPEG_VERSION="${{ matrix.ffmpeg_version }}" - export FFMPEG_ROOT="${PWD}/ffmpeg" - .github/scripts/ffmpeg/build.sh - - tar -cf ffmpeg.tar.gz ffmpeg/include ffmpeg/lib - - artifact_dir="${RUNNER_ARTIFACT_DIR}/$(date +%Y-%m-%d)/linux_aarch64/" - mkdir -p "${artifact_dir}" - mv ffmpeg.tar.gz "${artifact_dir}/${FFMPEG_VERSION}.tar.gz" - - LGPL-macOS: - strategy: - fail-fast: false - matrix: - ffmpeg_version: ["4.4.4", "5.1.4", "6.1.1", "master"] - runner: ["macos-m1-stable", "macos-12"] - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - with: - job-name: Build - upload-artifact: ffmpeg-lgpl - repository: pytorch/audio - runner: "${{ matrix.runner }}" - script: | - export FFMPEG_VERSION="${{ matrix.ffmpeg_version }}" - export FFMPEG_ROOT="${PWD}/ffmpeg" - .github/scripts/ffmpeg/build.sh - - tar -cf ffmpeg.tar.gz ffmpeg/include ffmpeg/lib - - artifact_dir="${RUNNER_ARTIFACT_DIR}/$(date +%Y-%m-%d)/macos_$(uname -m)" - mkdir -p "${artifact_dir}" - mv ffmpeg.tar.gz "${artifact_dir}/${FFMPEG_VERSION}.tar.gz" - - LGPL-Windows: - strategy: - fail-fast: false - matrix: - ffmpeg_version: ["4.4.4", "5.1.4", "6.1.1", "master"] - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - with: - job-name: Build - upload-artifact: ffmpeg-lgpl - repository: pytorch/audio - script: | - export FFMPEG_VERSION="${{ matrix.ffmpeg_version }}" - export FFMPEG_ROOT="${PWD}/ffmpeg" - .github/scripts/ffmpeg/build.bat - - tar -cf ffmpeg.tar.gz ffmpeg/include ffmpeg/bin - - artifact_dir="${RUNNER_ARTIFACT_DIR}/$(date +%Y-%m-%d)/windows" - mkdir -p "${artifact_dir}" - mv ffmpeg.tar.gz "${artifact_dir}/${FFMPEG_VERSION}.tar.gz" From d88178f93182139927dfb5a2b0fc94b180b903cc Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Thu, 11 Sep 2025 19:05:55 +0000 Subject: [PATCH 06/11] Revise linux gpu tests file --- .github/workflows/unittest-linux-gpu.yml | 260 +++++++---------------- 1 file changed, 76 insertions(+), 184 deletions(-) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index 7d6ca0e477..dd987d5e9c 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -1,189 +1,81 @@ name: Unit-tests on Linux GPU on: - pull_request: - push: - branches: - - nightly - - main - - release/* - workflow_dispatch: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: jobs: - build: - # Do not use matrix here to parameterize Python/CUDA versions. - # This job is required to pass for each PR. - # The name of the required job is sensitive to matrix parameter. - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - with: - job-name: Run tests - runner: linux.g5.4xlarge.nvidia.gpu - repository: pytorch/audio - gpu-arch-type: cuda - gpu-arch-version: "12.6" # See GPU_ARCH_ID below - timeout: 120 - - script: | - set -ex - - # Set up Environment Variables - export PYTHON_VERSION="3.10" - export CU_VERSION="11.8" - export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}" - export PIP_PROGRESS_BAR=off - export CONDA_QUIET=1 - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true - export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true - # Avoid reproducibility errors with CUBLAS: https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility - export CUBLAS_WORKSPACE_CONFIG=:4096:8 - - # Set CHANNEL - if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then - export CHANNEL=test - export BUILD_VERSION="$( cut -f 1 -d a version.txt )" - else - export CHANNEL=nightly - export BUILD_VERSION="$( cut -f 1 -d a version.txt )".dev"$(date "+%Y%m%d")" - fi - - # Install miniforge - wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" - bash Miniforge3.sh -b -p "${HOME}/conda" - source "${HOME}/conda/etc/profile.d/conda.sh" - - echo "::group::Create conda env" - # Mark Build Directory Safe - git config --global --add safe.directory /__w/audio/audio - conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}" - conda activate ./ci_env - conda install -q -y pip numpy - - echo "::endgroup::" - echo "::group::Install PyTorch" - - GPU_ARCH_ID=cu126 # This is hard-coded and must be consistent with gpu-arch-version. - PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL}/${GPU_ARCH_ID}" - pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}" - - echo "::endgroup::" - echo "::group::Install TorchAudio" - conda install --quiet --yes cmake>=3.18.0 ninja - pip install --progress-bar off -v . --no-build-isolation - - # TODO: Need to rely on torchcodec instead of building ffmpeg from source. - echo "::endgroup::" - echo "::group::Build FFmpeg" - conda install -q -y "ffmpeg<=7" - - echo "::endgroup::" - echo "::group::Install other dependencies" - - pip install --progress-bar off parameterized requests coverage pytest pytest-cov scipy numpy expecttest - - declare -a args=( - '-v' - '--cov=torchaudio' - "--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml" - '--durations' '100' - '-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency' - ) - - cd test - python3 -m torch.utils.collect_env - env | grep TORCHAUDIO || true - pytest "${args[@]}" torchaudio_unittest - coverage html - - commit-main: - if: github.ref_name == 'main' - permissions: - # Required for `git push` - # Note: - # This is not effective from fork. - # When you debug this, make sure to make a branch on pytorch and - # make PR from there. - contents: write - runs-on: ubuntu-latest - needs: build - steps: - - uses: actions/checkout@v3 - with: - ref: gh-pages - fetch-depth: 5 - - uses: actions/download-artifact@v3 - with: - name: docs - - name: Update main doc - run: | - set -x - - git config user.name "pytorchbot" - git config user.email "soumith+bot@pytorch.org" - # When `git clone`, `gh-pages` branch is fetched by default. - # The size of gh-pages grows significantly, so we use ammend and force push - # We add a new commit once a week - if [ "$(date +%d)" = "1" ]; then - git commit --allow-empty -m "placeholder" - fi - - # TODO: add tag-based process (need to handle the main directory name) - # Update the main doc - rm -rf main - mv html main - git add --all main || true - - git commit --amend -m "auto-generating sphinx docs" || true - git push -f - - # Push for release - # Make sure that version.txt is updated first (alpha suffix is removed) - commit-release: - if: startsWith(github.ref_name, 'release/') && ( github.event_name == 'workflow_dispatch' ) - permissions: - # Required for `git push` - # Note: - # This is not effective from fork. - # When you debug this, make sure to make a branch on pytorch and - # make PR from there. - contents: write - runs-on: ubuntu-latest - needs: build - steps: - - uses: actions/checkout@v3 - with: - ref: gh-pages - fetch-depth: 5 - - uses: actions/checkout@v4 - with: - path: _src - - uses: actions/download-artifact@v3 - with: - name: docs - - name: Update doc - run: | - set -x - - git config user.name "pytorchbot" - git config user.email "soumith+bot@pytorch.org" - # When `git clone`, `gh-pages` branch is fetched by default. - # The size of gh-pages grows significantly, so we use ammend and force push - # We add a new commit once a week - if [ "$(date +%d)" = "1" ]; then - git commit --allow-empty -m "placeholder" - fi - - dirname="$(cat _src/version.txt)" - rm -rf "${dirname}" - mv html "${dirname}" - git add --all "${dirname}" || true - - git commit --amend -m "auto-generating sphinx docs" || true - git push -f + build: + strategy: + matrix: + python-version: ["3.11"] + cuda-version: ["12.6"] + ffmpeg-version: ["7"] + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + with: + runner: linux.4xlarge.nvidia.gpu + repository: pytorch/audio + gpu-arch-type: cuda + gpu-arch-version: ${{ matrix.cuda-version }} + timeout: 120 + job-name: Run tests + + script: | + set -ex + # Set up Environment Variables + export PYTHON_VERSION="${{ matrix.python-version }}" + export FFMPEG_VERSION="${{ matrix.ffmpeg-version }}" + export CU_VERSION_WITHOUT_PERIODS="$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')" + export PIP_PROGRESS_BAR=off + export CONDA_QUIET=1 + + # Set UPLOAD_CHANNEL + if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + export UPLOAD_CHANNEL=test + else + export UPLOAD_CHANNEL=nightly + fi + + echo "::group::Create conda env" + # Mark Build Directory Safe + git config --global --add safe.directory /__w/audio/audio + conda create -c conda-forge --strict-channel-priority -y --prefix ci_env python="${PYTHON_VERSION}" ffmpeg="${FFMPEG_VERSION}" cmake ninja + conda activate ./ci_env + conda info + python -m pip install --upgrade pip + echo "::endgroup::" + + echo "::group::Install PyTorch and TorchCodec" + PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/cu${CU_VERSION_WITHOUT_PERIODS}" + python -m pip install numpy + python -m pip install --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}" + python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + echo "::endgroup::" + + echo "::group::Build and install TorchAudio" + python -m pip install . -v --no-build-isolation + echo "::endgroup::" + + pip install --progress-bar off parameterized requests coverage pytest pytest-cov scipy numpy expecttest + declare -a args=( + '-v' + '--cov=torchaudio' + "--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml" + '--durations' '100' + '-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency' + ) + + cd test + python3 -m torch.utils.collect_env + env | grep TORCHAUDIO || true + pytest "${args[@]}" torchaudio_unittest + coverage html From 83d37328aa59a3022810a1ca6e9591b5db40f2b5 Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Thu, 11 Sep 2025 19:12:50 +0000 Subject: [PATCH 07/11] Make changes from code review --- .github/workflows/unittest-linux-gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index 6a65ce11b4..f325b42c57 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -66,7 +66,7 @@ jobs: echo "::endgroup::" echo "::group::Install Python Dependencies" - pip install --progress-bar off parameterized requests coverage pytest pytest-cov scipy numpy expecttest + pip install parameterized requests coverage pytest pytest-cov scipy expecttest echo "::endgroup::" echo "::group::Run tests" From 0be432e1e4ee20c58bcc44e53702b7c54a7f3ca8 Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Thu, 11 Sep 2025 21:31:57 +0000 Subject: [PATCH 08/11] Add test skip variables --- .github/workflows/unittest-linux-gpu.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index f325b42c57..0cfc519def 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -37,6 +37,13 @@ jobs: export CU_VERSION_WITHOUT_PERIODS="$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')" export PIP_PROGRESS_BAR=off export CONDA_QUIET=1 + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true + export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true # Set UPLOAD_CHANNEL if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then From 0990d5091d3ade212c6cc641573d0c6ef4722132 Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Fri, 12 Sep 2025 01:00:22 +0000 Subject: [PATCH 09/11] Add back deterministic workspace config --- .github/workflows/unittest-linux-gpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index 0cfc519def..d9266b9746 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -44,6 +44,7 @@ jobs: export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true + export CUBLAS_WORKSPACE_CONFIG=:4096:8 # Set UPLOAD_CHANNEL if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then From cd912e6a3713057f2d2960bcaf952d9c866e940f Mon Sep 17 00:00:00 2001 From: Sam Anklesaria Date: Fri, 12 Sep 2025 02:34:28 +0000 Subject: [PATCH 10/11] Let deterministic settings use less cuda memory --- .github/workflows/unittest-linux-gpu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index d9266b9746..d29193a31f 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -44,7 +44,7 @@ jobs: export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true - export CUBLAS_WORKSPACE_CONFIG=:4096:8 + export CUBLAS_WORKSPACE_CONFIG=:16:8 # Set UPLOAD_CHANNEL if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then From 5b30c6468584561a27f88563f2e1e12ac3c8f1b8 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Fri, 12 Sep 2025 10:52:30 +0300 Subject: [PATCH 11/11] Use runner linux.g5.4xlarge.nvidia.gpu --- .github/workflows/unittest-linux-gpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml index d29193a31f..f3c4da3eb5 100644 --- a/.github/workflows/unittest-linux-gpu.yml +++ b/.github/workflows/unittest-linux-gpu.yml @@ -22,7 +22,7 @@ jobs: id-token: write contents: read with: - runner: linux.4xlarge.nvidia.gpu + runner: linux.g5.4xlarge.nvidia.gpu repository: pytorch/audio gpu-arch-type: cuda gpu-arch-version: ${{ matrix.cuda-version }} @@ -85,6 +85,7 @@ jobs: "--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml" '--durations' '100' '-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency' + '-x' ) cd test