ci: reduce number of jobs & improve coverage measurement (#5052)

davidhewitt · web-flow · commit 7d166f70b451 · 2025-04-11T15:43:51.000Z
* ci: run coverage as part of "build" job

* merge free-threaded tests into main build jobs

* make `test-introspection` jobs only run on PRs if requested

* fix 3.7 &amp; 3.8 build error

* fix `check-feature-powerset`
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -27,6 +27,15 @@ jobs:
     if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          # For PRs, we need to run on the real PR head, not the resultant merge of the PR into the target branch.
+          #
+          # This is necessary for coverage reporting to make sense; we then get exactly the coverage change
+          # between the base branch and the real PR head.
+          #
+          # If it were run on the merge commit the problem is that the coverage potentially does not align
+          # with the commit diff, because the merge may affect line numbers.
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
 
       - name: Set up Python ${{ inputs.python-version }}
         uses: actions/setup-python@v5
@@ -40,7 +49,7 @@ jobs:
       - name: Install nox
         run: python -m pip install --upgrade pip && pip install nox
 
-      - if: inputs.python-version == 'graalpy24.1'
+      - if: ${{ startsWith(inputs.python-version, 'graalpy24.1') }}
         name: Install GraalPy virtualenv (only GraalPy 24.1)
         run: python -m pip install 'git+https://github.com/oracle/graalpython#egg=graalpy_virtualenv_seeder&subdirectory=graalpy_virtualenv_seeder'
 
@@ -49,8 +58,17 @@ jobs:
         with:
           toolchain: ${{ inputs.rust }}
           targets: ${{ inputs.rust-target }}
-          # needed to correctly format errors, see #1865
-          components: rust-src
+          # rust-src needed to correctly format errors, see #1865
+          components: rust-src,llvm-tools-preview
+
+      # On windows 32 bit, we are running on an x64 host, so we need to specifically set the target
+      # NB we don't do this for *all* jobs because it breaks coverage of proc macros to have an
+      # explicit target set.
+      - name: Set Rust target for Windows 32-bit
+        if: inputs.os == 'windows-latest' && inputs.python-architecture == 'x86'
+        shell: bash
+        run: |
+          echo "CARGO_BUILD_TARGET=i686-pc-windows-msvc" >> $GITHUB_ENV
 
       - uses: Swatinem/rust-cache@v2
         with:
@@ -72,6 +90,16 @@ jobs:
         name: Prepare to test on nightly rust
         run: echo "MAYBE_NIGHTLY=nightly" >> "$GITHUB_ENV"
 
+      - if: ${{ github.event_name != 'merge_group' }}
+        name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+
+      - if: ${{ github.event_name != 'merge_group' }}
+        name: Prepare coverage environment
+        run: |
+          cargo llvm-cov clean --workspace --profraw-only
+          nox -s set-coverage-env
+
       - name: Build docs
         run: nox -s docs
 
@@ -154,9 +182,26 @@ jobs:
         if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }}
         run: nox -s ffi-check
 
+      - if: ${{ github.event_name != 'merge_group' }}
+        name: Generate coverage report
+        run: cargo llvm-cov
+          --package=pyo3
+          --package=pyo3-build-config
+          --package=pyo3-macros-backend
+          --package=pyo3-macros
+          --package=pyo3-ffi
+          report --codecov --output-path coverage.json
+
+      - if: ${{ github.event_name != 'merge_group' }}
+        name: Upload coverage report
+        uses: codecov/codecov-action@v4
+        with:
+          file: coverage.json
+          name: ${{ inputs.os }}/${{ inputs.python-version }}/${{ inputs.rust }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+
     env:
       CARGO_TERM_VERBOSE: true
-      CARGO_BUILD_TARGET: ${{ inputs.rust-target }}
       RUST_BACKTRACE: 1
       RUSTFLAGS: "-D warnings"
       RUSTDOCFLAGS: "-D warnings"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -225,6 +225,17 @@ jobs:
                 python-architecture: "x64",
                 rust-target: "x86_64-unknown-linux-gnu",
               }
+          # Also test free-threaded Python just for latest Python version, on ubuntu
+          # (run for all OSes on build-full)
+          - rust: stable
+            python-version: "3.13t"
+            platform:
+              {
+                os: "ubuntu-latest",
+                python-architecture: "x64",
+                rust-target: "x86_64-unknown-linux-gnu",
+              }
+
   build-full:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
     name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }}
@@ -252,6 +263,7 @@ jobs:
             "3.11",
             "3.12",
             "3.13",
+            "3.13t",
             "pypy3.9",
             "pypy3.10",
             "pypy3.11",
@@ -460,35 +472,6 @@ jobs:
           components: rust-src
       - run: cargo rustdoc --lib --no-default-features --features full,jiff-02 -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]"
 
-  coverage:
-    if: ${{ github.event_name != 'merge_group' }}
-    needs: [fmt]
-    name: coverage ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: ["windows-latest", "macos-latest", "ubuntu-latest"]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-      - uses: Swatinem/rust-cache@v2
-        with:
-          save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: llvm-tools-preview,rust-src
-      - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@cargo-llvm-cov
-      - run: python -m pip install --upgrade pip && pip install nox
-      - run: nox -s coverage
-      - uses: codecov/codecov-action@v5
-        with:
-          files: coverage.json
-          name: ${{ matrix.os }}
-          token: ${{ secrets.CODECOV_TOKEN }}
-
   emscripten:
     name: emscripten
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
@@ -575,43 +558,6 @@ jobs:
           echo PYO3_CONFIG_FILE=$PYO3_CONFIG_FILE >> $GITHUB_ENV
       - run: python3 -m nox -s test
 
-  test-free-threaded:
-    needs: [fmt]
-    name: Free threaded tests - ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: rust-src
-      - uses: actions/setup-python@v5.5.0
-        with:
-          python-version: "3.13t"
-      - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@cargo-llvm-cov
-      - run: python3 -m sysconfig
-      - run: python3 -m pip install --upgrade pip && pip install nox
-      - name: Prepare coverage environment
-        run: |
-          cargo llvm-cov clean --workspace --profraw-only
-          nox -s set-coverage-env
-      - run: nox -s ffi-check
-      - run: nox
-      - name: Generate coverage report
-        run: nox -s generate-coverage-report
-      - name: Upload coverage report
-        uses: codecov/codecov-action@v5
-        with:
-          files: coverage.json
-          name: ${{ matrix.os }}-test-free-threaded
-          token: ${{ secrets.CODECOV_TOKEN }}
-
   test-version-limits:
     needs: [fmt]
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
@@ -758,30 +704,32 @@ jobs:
 
   test-introspection:
     needs: [fmt]
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-test-introspection') || contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
     strategy:
       matrix:
-        platform: [
-          {
-            os: "macos-latest",
-            python-architecture: "arm64",
-            rust-target: "aarch64-apple-darwin",
-          },
-          {
-            os: "ubuntu-latest",
-            python-architecture: "x64",
-            rust-target: "x86_64-unknown-linux-gnu",
-          },
-          {
-            os: "windows-latest",
-            python-architecture: "x64",
-            rust-target: "x86_64-pc-windows-msvc",
-          },
-          {
-            os: "windows-latest",
-            python-architecture: "x86",
-            rust-target: "i686-pc-windows-msvc",
-          },
-        ]
+        platform:
+          [
+            {
+              os: "macos-latest",
+              python-architecture: "arm64",
+              rust-target: "aarch64-apple-darwin",
+            },
+            {
+              os: "ubuntu-latest",
+              python-architecture: "x64",
+              rust-target: "x86_64-unknown-linux-gnu",
+            },
+            {
+              os: "windows-latest",
+              python-architecture: "x64",
+              rust-target: "x86_64-pc-windows-msvc",
+            },
+            {
+              os: "windows-latest",
+              python-architecture: "x86",
+              rust-target: "i686-pc-windows-msvc",
+            },
+          ]
     runs-on: ${{ matrix.platform.os }}
     steps:
       - uses: actions/checkout@v4
@@ -811,10 +759,8 @@ jobs:
       - valgrind
       - careful
       - docsrs
-      - coverage
       - emscripten
       - test-debug
-      - test-free-threaded
       - test-version-limits
       - check-feature-powerset
       - test-cross-compilation
diff --git a/examples/setuptools-rust-starter/noxfile.py b/examples/setuptools-rust-starter/noxfile.py
@@ -1,8 +1,11 @@
 import nox
+import sys
 
 
 @nox.session
 def python(session: nox.Session):
+    if sys.version_info < (3, 9):
+        session.skip("Python 3.9 or later is required for setuptools-rust 1.11")
     session.env["SETUPTOOLS_RUST_CARGO_PROFILE"] = "dev"
     session.install(".[dev]")
     session.run("pytest")
diff --git a/noxfile.py b/noxfile.py
@@ -72,6 +72,9 @@ def _supported_interpreter_versions(
     min_minor = int(min_version.split(".")[1])
     max_minor = int(max_version.split(".")[1])
     versions = [f"{major}.{minor}" for minor in range(min_minor, max_minor + 1)]
+    # Add free-threaded builds for 3.13+
+    if python_impl == "cpython":
+        versions += [f"{major}.{minor}t" for minor in range(13, max_minor + 1)]
     return versions
 
 
@@ -725,7 +728,10 @@ def check_feature_powerset(session: nox.Session):
 
     cargo_toml = toml.loads((PYO3_DIR / "Cargo.toml").read_text())
 
-    EXPECTED_ABI3_FEATURES = {f"abi3-py3{ver.split('.')[1]}" for ver in PY_VERSIONS}
+    # free-threaded builds do not support ABI3 (yet)
+    EXPECTED_ABI3_FEATURES = {
+        f"abi3-py3{ver.split('.')[1]}" for ver in PY_VERSIONS if not ver.endswith("t")
+    }
 
     EXCLUDED_FROM_FULL = {
         "nightly",
