refactor: make backend installs transactional #56
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build PyWhisperCPP Wheels | |
| # CUDA wheels only; CPU comes from PyPI. Version, commit, and variant suffix are | |
| # derived from lib/src/backend_installer.py so the workflow never restates the | |
| # contract. Wheels publish to the rolling `wheels-v2` release the installer | |
| # downloads from (WHEEL_BASE_URL). Pushes to main are gated by a preflight that | |
| # skips the build when the pinned version's wheels are already published, so a | |
| # version bump ships wheels automatically and unrelated installer edits cost | |
| # one curl. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pywhispercpp_commit: | |
| description: 'PyWhisperCPP commit (empty = backend_installer.PYWHISPERCPP_PINNED_COMMIT)' | |
| required: false | |
| default: '' | |
| push: | |
| branches: [main] | |
| paths: ['lib/src/backend_installer.py'] | |
| concurrency: | |
| group: build-wheels | |
| cancel-in-progress: false | |
| env: | |
| CUDA_BUILD_VERSION: '12.6' # one 12.x build serves all CUDA 12 | |
| jobs: | |
| preflight: | |
| name: Check wheels-v2 for pinned version | |
| runs-on: ubuntu-latest | |
| outputs: | |
| build: ${{ steps.gate.outputs.build }} | |
| steps: | |
| - name: Checkout hyprwhspr | |
| uses: actions/checkout@v4 | |
| - name: Skip if installer's wheels already published | |
| id: gate | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "build=true" >> "$GITHUB_OUTPUT" | |
| echo "manual run: building" | |
| exit 0 | |
| fi | |
| probe=$(PYTHONPATH=lib/src python3 -c "import backend_installer as b; print(b._get_wheel_filename('3.10', 'cuda12', True))") | |
| url="https://github.com/${GITHUB_REPOSITORY}/releases/download/wheels-v2/${probe}" | |
| code=$(curl -s -o /dev/null -w '%{http_code}' "$url") | |
| if [ "$code" = "302" ]; then | |
| echo "build=false" >> "$GITHUB_OUTPUT" | |
| echo "already on wheels-v2: $probe - nothing to do" | |
| else | |
| echo "build=true" >> "$GITHUB_OUTPUT" | |
| echo "missing on wheels-v2 (HTTP $code): $probe - building" | |
| fi | |
| build-cuda: | |
| name: Build CUDA wheel (Python ${{ matrix.python-version }}) | |
| needs: preflight | |
| if: needs.preflight.outputs.build == 'true' | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] | |
| steps: | |
| - name: Checkout hyprwhspr | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| allow-prereleases: true | |
| - name: Derive installer contract (from backend_installer.py) | |
| run: | | |
| ver=$(PYTHONPATH=lib/src python -c "import backend_installer as b; print(b.PYWHISPERCPP_VERSION)") | |
| pin=$(PYTHONPATH=lib/src python -c "import backend_installer as b; print(b.PYWHISPERCPP_PINNED_COMMIT)") | |
| variant=$(PYTHONPATH=lib/src python -c "import backend_installer as b; print(b._get_wheel_variant('${CUDA_BUILD_VERSION}'))") | |
| echo "EXPECTED_VERSION=$ver" >> "$GITHUB_ENV" | |
| echo "VARIANT=$variant" >> "$GITHUB_ENV" | |
| commit='${{ github.event.inputs.pywhispercpp_commit }}' | |
| [ -n "$commit" ] || commit="$pin" | |
| echo "PYWHISPERCPP_COMMIT=$commit" >> "$GITHUB_ENV" | |
| echo "contract: version=$ver variant=$variant commit=$commit" | |
| if [ "$variant" != "cuda12" ]; then | |
| echo "::error::Installer maps CUDA ${CUDA_BUILD_VERSION} to '$variant', expected 'cuda12'." | |
| exit 1 | |
| fi | |
| if [ "$commit" != "$pin" ]; then | |
| echo "::warning::Building $commit, installer pins $pin (v$ver)." | |
| fi | |
| - name: Install CUDA Toolkit ${{ env.CUDA_BUILD_VERSION }} | |
| uses: Jimver/cuda-toolkit@v0.2.35 | |
| id: cuda-toolkit | |
| with: | |
| cuda: ${{ env.CUDA_BUILD_VERSION }}.0 | |
| method: 'network' | |
| sub-packages: '["nvcc", "cudart-dev"]' | |
| - name: Install additional CUDA libraries | |
| run: | | |
| sudo apt-get update | |
| CUDA_VER_DASH="${CUDA_BUILD_VERSION//./-}" | |
| sudo apt-get install -y libcublas-dev-${CUDA_VER_DASH} | |
| - name: Install build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential cmake git | |
| pip install --upgrade pip wheel setuptools build | |
| - name: Clone pywhispercpp | |
| run: | | |
| git clone --recurse-submodules https://github.com/Absadiki/pywhispercpp.git pywhispercpp-src | |
| cd pywhispercpp-src | |
| git checkout ${{ env.PYWHISPERCPP_COMMIT }} | |
| git submodule update --init --recursive | |
| - name: Build wheel with CUDA support | |
| env: | |
| GGML_CUDA: 'ON' | |
| CUDACXX: ${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/bin/nvcc | |
| NO_REPAIR: '1' # CUDA driver libs come from the user's system | |
| run: | | |
| cd pywhispercpp-src | |
| pip wheel . --no-deps --wheel-dir ../dist | |
| - name: Assert built version matches installer | |
| run: | | |
| cd dist | |
| built="$(ls pywhispercpp-*.whl | head -n1)" | |
| version="$(echo "$built" | sed -E 's/^pywhispercpp-([^-]+)-.*/\1/')" | |
| if [ "$version" != "${EXPECTED_VERSION}" ]; then | |
| echo "::error::Built $version != installer's ${EXPECTED_VERSION}." | |
| exit 1 | |
| fi | |
| - name: Rename wheel with CUDA variant suffix | |
| run: | | |
| cd dist | |
| for f in *.whl; do mv "$f" "${f%.whl}+${VARIANT}.whl"; done | |
| ls -la | |
| - name: Upload wheel artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheel-cuda12-py${{ matrix.python-version }} | |
| path: dist/*.whl | |
| verify-wheels: | |
| # Derive the filename the installer will request and assert the build made it. | |
| name: Verify wheel (Python ${{ matrix.python-version }}) | |
| needs: [build-cuda] | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] | |
| steps: | |
| - name: Checkout hyprwhspr | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| allow-prereleases: true | |
| - name: Derive expected filename from installer | |
| run: | | |
| py='${{ matrix.python-version }}' | |
| variant=$(PYTHONPATH=lib/src python -c "import backend_installer as b; print(b._get_wheel_variant('${CUDA_BUILD_VERSION}'))") | |
| dl=$(PYTHONPATH=lib/src python -c "import backend_installer as b; print(b._get_wheel_filename('${py}', '${variant}', True))") | |
| pip=$(PYTHONPATH=lib/src python -c "import backend_installer as b; print(b._get_wheel_filename('${py}', '${variant}', False))") | |
| echo "EXPECTED_WHEEL=$dl" >> "$GITHUB_ENV" | |
| echo "WHEEL_PIP_NAME=$pip" >> "$GITHUB_ENV" | |
| echo "installer wants: $dl" | |
| - name: Download wheel artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel-cuda12-py${{ matrix.python-version }} | |
| path: dist | |
| - name: Assert build produced the requested file | |
| run: | | |
| if [ ! -f "dist/${EXPECTED_WHEEL}" ]; then | |
| echo "::error::Installer wants ${EXPECTED_WHEEL}; build did not produce it." | |
| ls -la dist | |
| exit 1 | |
| fi | |
| - name: Confirm wheel installs on this Python | |
| run: | | |
| cp "dist/${EXPECTED_WHEEL}" "dist/${WHEEL_PIP_NAME}" | |
| # --no-deps: validate cp tag only; import needs a GPU absent on CI. | |
| pip install --no-deps "dist/${WHEEL_PIP_NAME}" | |
| - name: Smoke-check native lib linkage | |
| run: | | |
| site=$(python -c "import sysconfig; print(sysconfig.get_paths()['platlib'])") | |
| sos=$(find "$site" -maxdepth 2 \( -name '*.so' -o -name '*.so.*' \) | grep -iE 'pywhispercpp|whisper|ggml' || true) | |
| [ -n "$sos" ] || { echo "::error::no pywhispercpp native libs found in $site"; exit 1; } | |
| # Point the loader at the bundle dirs so the wheel's own libs (libggml*, libwhisper) | |
| # resolve each other; only genuinely-absent CUDA libs (no GPU on CI) should remain. | |
| libdirs=$(printf '%s\n' $sos | xargs -n1 dirname | sort -u | paste -sd: -) | |
| missing=$(LD_LIBRARY_PATH="$libdirs" ldd $sos 2>/dev/null | grep 'not found' | grep -viE 'libcu|libnv' | sort -u || true) | |
| if [ -n "$missing" ]; then | |
| echo "::error::Non-CUDA shared libs unresolved (wheel likely unusable):" | |
| echo "$missing" | |
| exit 1 | |
| fi | |
| echo "OK: only CUDA libs unresolved (expected without a GPU)" | |
| publish-wheels: | |
| name: Publish wheels to wheels-v2 | |
| needs: [build-cuda, verify-wheels] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Download all wheel artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: wheels | |
| pattern: wheel-* | |
| merge-multiple: true | |
| - name: Upload to rolling wheels-v2 release (merged checksums) | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| ls -la wheels/ | |
| (cd wheels && sha256sum *.whl) > new-sums.txt | |
| # Merge with the release's existing checksums: replace entries for | |
| # files being (re)uploaded, keep every other version's entries. | |
| gh release download wheels-v2 --repo "$GITHUB_REPOSITORY" \ | |
| --pattern SHA256SUMS.txt --output existing-sums.txt || : > existing-sums.txt | |
| awk '{print $2}' new-sums.txt > new-files.txt | |
| grep -vFf new-files.txt existing-sums.txt > SHA256SUMS.txt || true | |
| cat new-sums.txt >> SHA256SUMS.txt | |
| sort -k2 -o SHA256SUMS.txt SHA256SUMS.txt | |
| cat SHA256SUMS.txt | |
| gh release upload wheels-v2 wheels/*.whl SHA256SUMS.txt \ | |
| --repo "$GITHUB_REPOSITORY" --clobber | |
| cp SHA256SUMS.txt wheels/ | |
| - name: Upload wheels as workflow artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: all-wheels | |
| path: wheels/ |