diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml deleted file mode 100644 index 84da2199..00000000 --- a/.github/workflows/CI.yml +++ /dev/null @@ -1,181 +0,0 @@ -# This file is autogenerated by maturin v1.7.4 -# To update, run -# -# maturin generate-ci github -m bindings/python/Cargo.toml -# -name: CI - -on: - push: - branches: - - main - - master - tags: - - '*' - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: ubuntu-latest - target: x86_64 - - runner: ubuntu-latest - target: x86 - - runner: ubuntu-latest - target: aarch64 - - runner: ubuntu-latest - target: armv7 - - runner: ubuntu-latest - target: s390x - - runner: ubuntu-latest - target: ppc64le - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path bindings/python/Cargo.toml - sccache: 'true' - manylinux: auto - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-linux-${{ matrix.platform.target }} - path: dist - - musllinux: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: ubuntu-latest - target: x86_64 - - runner: ubuntu-latest - target: x86 - - runner: ubuntu-latest - target: aarch64 - - runner: ubuntu-latest - target: armv7 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path bindings/python/Cargo.toml - sccache: 'true' - manylinux: musllinux_1_2 - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-musllinux-${{ matrix.platform.target }} - path: dist - - windows: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: windows-latest - target: x64 - - runner: windows-latest - target: x86 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - architecture: ${{ matrix.platform.target }} - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path bindings/python/Cargo.toml - sccache: 'true' - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-windows-${{ matrix.platform.target }} - path: dist - - macos: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: macos-12 - target: x86_64 - - runner: macos-14 - target: aarch64 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path bindings/python/Cargo.toml - sccache: 'true' - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-macos-${{ matrix.platform.target }} - path: dist - - sdist: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Build sdist - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out dist --manifest-path bindings/python/Cargo.toml - - name: Upload sdist - uses: actions/upload-artifact@v4 - with: - name: wheels-sdist - path: dist - - release: - name: Release - runs-on: ubuntu-latest - if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - needs: [linux, musllinux, windows, macos, sdist] - permissions: - # Use to sign the release artifacts - id-token: write - # Used to upload release artifacts - contents: write - # Used to generate artifact attestation - attestations: write - steps: - - uses: actions/download-artifact@v4 - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-path: 'wheels-*/*' - - name: Publish to PyPI - if: "startsWith(github.ref, 'refs/tags/')" - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST}} - with: - command: upload - args: --non-interactive --skip-existing wheels-*/* diff --git a/.github/workflows/python-bench.yml b/.github/workflows/python-bench.yml index b97ecafb..fafeddbe 100644 --- a/.github/workflows/python-bench.yml +++ b/.github/workflows/python-bench.yml @@ -4,6 +4,7 @@ on: branches: - main + permissions: # deployments permission to deploy GitHub pages website deployments: write diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 9393714d..84da2199 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,192 +1,181 @@ -name: Python Release +# This file is autogenerated by maturin v1.7.4 +# To update, run +# +# maturin generate-ci github -m bindings/python/Cargo.toml +# +name: CI + on: push: + branches: + - main + - master tags: - - v* + - '*' + pull_request: + workflow_dispatch: -env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }} - DIST_DIR: ${{ github.sha }} +permissions: + contents: read jobs: - lock_exists: - runs-on: ubuntu-latest - name: Cargo.lock - steps: - - uses: actions/checkout@v3 - - name: Cargo.lock lock exists - run: cat Cargo.lock - working-directory: ./bindings/python - - build: - name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }}) - # only run on push to main and on release - needs: [lock_exists] - - if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') - outputs: - artifact: ${{ steps.artifact-upload-step.outputs.artifact-id }} + linux: + runs-on: ${{ matrix.platform.runner }} strategy: - fail-fast: false matrix: - os: [ubuntu, macos, windows] - target: [x86_64, aarch64] - manylinux: [auto] - include: - - os: ubuntu - platform: linux - - os: windows - ls: dir - interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9 pypy3.10 - - os: windows - ls: dir - target: i686 - python-architecture: x86 - interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - - os: windows - ls: dir + platform: + - runner: ubuntu-latest target: x86_64 - python-architecture: x64 - interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - # - os: windows - # ls: dir - # target: aarch64 - # interpreter: 3.11 3.12 - - os: macos - target: aarch64 - interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.8 pypy3.9 pypy3.10 - - os: ubuntu - platform: linux - target: i686 - - os: ubuntu - platform: linux + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest target: aarch64 - - - os: ubuntu - platform: linux + - runner: ubuntu-latest target: armv7 - interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 3.13 - # musllinux - - os: ubuntu - platform: linux - target: x86_64 - manylinux: musllinux_1_1 - - os: ubuntu - platform: linux - target: aarch64 - manylinux: musllinux_1_1 - - os: ubuntu - platform: linux - target: ppc64le - interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 3.13 - - os: ubuntu - platform: linux + - runner: ubuntu-latest target: s390x - interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 3.13 - exclude: - # Optimized PGO builds for x86_64 manylinux and windows follow a different matrix, - # maybe in future maturin-action can support this automatically - # - os: ubuntu - # target: x86_64 - # manylinux: auto - # - os: windows - # target: x86_64 - # Windows on arm64 only supports Python 3.11+ - - os: windows - target: aarch64 - - runs-on: ${{ matrix.os }}-latest + - runner: ubuntu-latest + target: ppc64le steps: - - uses: actions/checkout@v3 - - - name: set up python - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: ${{ matrix.python-architecture || 'x64' }} - - - run: pip install -U twine - - - name: build wheels + python-version: 3.x + - name: Build wheels uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.target }} - working-directory: ./bindings/python - manylinux: ${{ matrix.manylinux || 'auto' }} - container: ${{ matrix.container }} - args: --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 3.12 3.13 pypy3.7 pypy3.8 pypy3.9 pypy3.10' }} ${{ matrix.extra-build-args }} - rust-toolchain: stable - docker-options: -e CI + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path bindings/python/Cargo.toml + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: dist - - run: ${{ matrix.ls || 'ls -lh' }} dist/ - working-directory: ./bindings/python + musllinux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: x86 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: armv7 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path bindings/python/Cargo.toml + sccache: 'true' + manylinux: musllinux_1_2 + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.platform.target }} + path: dist - - run: twine check --strict dist/* - working-directory: ./bindings/python + windows: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: windows-latest + target: x64 + - runner: windows-latest + target: x86 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + architecture: ${{ matrix.platform.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path bindings/python/Cargo.toml + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-windows-${{ matrix.platform.target }} + path: dist - # - name: Upload wheels - # shell: bash - # run: | - # pip install awscli - # aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" + macos: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: macos-12 + target: x86_64 + - runner: macos-14 + target: aarch64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path bindings/python/Cargo.toml + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: dist - - uses: actions/upload-artifact@v4 - id: artifact-upload-step - with: - name: pypi_files-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux }} - path: bindings/python/dist - build-sdist: - name: build sdist - needs: [lock_exists] + sdist: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: PyO3/maturin-action@v1 + - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 with: - working-directory: ./bindings/python command: sdist - args: --out dist - rust-toolchain: stable - # - name: Upload wheels - # shell: bash - # run: | - # pip install awscli - # aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" - - uses: actions/upload-artifact@v4 - with: - name: pypi_files-src - path: ./bindings/python/dist - + args: --out dist --manifest-path bindings/python/Cargo.toml + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist - upload_package: - name: Upload package to PyPi + release: + name: Release runs-on: ubuntu-latest - needs: [build, build-sdist] - + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: [linux, musllinux, windows, macos, sdist] + permissions: + # Use to sign the release artifacts + id-token: write + # Used to upload release artifacts + contents: write + # Used to generate artifact attestation + attestations: write steps: - - uses: actions/checkout@v3 - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.12" - architecture: x64 - - # - name: Retrieve all wheels - # shell: bash - # run: | - # pip install awscli - # aws s3 sync "s3://safetensors-releases/python/$DIST_DIR" ./bindings/python/dist - uses: actions/download-artifact@v4 + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 with: - path: ./bindings/python/dist - merge-multiple: true - - - name: Upload to PyPi - working-directory: ./bindings/python - run: | - pip install twine - twine upload dist/* -u __token__ -p "$PYPI_TOKEN" + subject-path: 'wheels-*/*' + - name: Publish to PyPI + if: "startsWith(github.ref, 'refs/tags/')" + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST}} + with: + command: upload + args: --non-interactive --skip-existing wheels-*/* diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 306da132..74e0bad7 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -42,7 +42,7 @@ tensorflow = [ # pinning tf version 2.11.0 for doc-builder pinned-tf = [ "safetensors[numpy]", - "tensorflow==2.11.0", + "tensorflow==2.18.0", ] jax = [ "safetensors[numpy]",