Skip to content

CI/CD

CI/CD #1435

Workflow file for this run

---
name: CI/CD
on:
merge_group:
push:
branches:
- master
- >-
[0-9].[0-9]+
tags:
- v*
pull_request:
branches:
- master
- >-
[0-9].[0-9]+
schedule:
- cron: 0 6 * * * # Daily 6AM UTC build
concurrency:
group: ${{ github.ref }}-${{ github.workflow }}
cancel-in-progress: true
env:
COLOR: >- # Supposedly, pytest or coveragepy use this
yes
FORCE_COLOR: 1 # Request colored output from CLI tools supporting it
MYPY_FORCE_COLOR: 1 # MyPy's color enforcement
PIP_DISABLE_PIP_VERSION_CHECK: 1
PIP_NO_PYTHON_VERSION_WARNING: 1
PIP_NO_WARN_SCRIPT_LOCATION: 1
PRE_COMMIT_COLOR: always
PROJECT_NAME: propcache
PY_COLORS: 1 # Recognized by the `py` package, dependency of `pytest`
PYTHONIOENCODING: utf-8
PYTHONUTF8: 1
PYTHON_LATEST: 3.12
TOX_PARALLEL_NO_SPINNER: 1 # Disable tox's parallel run spinner animation
UV_PYTHON_PREFERENCE: only-managed # avoid ancient system Python
TOX_TESTENV_PASSENV: >- # Make tox-wrapped tools see color requests
CI
COLOR
FORCE_COLOR
GITHUB_*
MYPY_FORCE_COLOR
NO_COLOR
PIP_DISABLE_PIP_VERSION_CHECK
PIP_NO_PYTHON_VERSION_WARNING
PIP_NO_WARN_SCRIPT_LOCATION
PRE_COMMIT_COLOR
PY_COLORS
PYTEST_THEME
PYTEST_THEME_MODE
PYTHONDONTWRITEBYTECODE
PYTHONIOENCODING
PYTHONLEGACYWINDOWSSTDIO
PYTHONTRACEMALLOC
PYTHONUTF8
UPSTREAM_REPOSITORY_ID: >-
866565706
jobs:
pre-setup:
name: ⚙️ Pre-set global build settings
runs-on: ubuntu-latest
timeout-minutes: 2
defaults:
run:
shell: python
outputs:
# NOTE: These aren't env vars because the `${{ env }}` context is
# NOTE: inaccessible when passing inputs to reusable workflows.
cache-key-for-dep-files: >-
${{ steps.calc-cache-key-files.outputs.cache-key-for-dep-files }}
dists-artifact-name: python-package-distributions
dist-version: ${{ steps.dist-version.outputs.dist-version }}
project-name: ${{ env.PROJECT_NAME }}
sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }}
pure-python-wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }}
upstream-repository-id: ${{ env.UPSTREAM_REPOSITORY_ID }}
steps:
- name: Switch to using Python ${{ env.PYTHON_LATEST }} by default
uses: actions/setup-python@v6
with:
python-version: ${{ env.PYTHON_LATEST }}
- name: Check out src from Git
uses: actions/checkout@v6
- name: >-
Calculate dependency files' combined hash value
for use in the cache key
id: calc-cache-key-files
uses: ./.github/actions/cache-keys
- name: Read the package version from the source tree
id: dist-version
run: |
import re
from os import environ
from pathlib import Path
FILE_APPEND_MODE = 'a'
VERSION_RE = re.compile(
r'''^__version__\s*=\s*["']([^"']+)["']''',
re.M,
)
init_text = Path('src/propcache/__init__.py').read_text()
match = VERSION_RE.search(init_text)
if match is None:
raise SystemExit(
'Could not find __version__ in propcache/__init__.py',
)
with Path(environ['GITHUB_OUTPUT']).open(mode=FILE_APPEND_MODE) as out:
print(f'dist-version={match.group(1)}', file=out)
- name: Set the expected dist artifact names
id: artifact-name
env:
DIST_VERSION: ${{ steps.dist-version.outputs.dist-version }}
run: |
from os import environ
from pathlib import Path
FILE_APPEND_MODE = 'a'
project_name = '${{ env.PROJECT_NAME }}'
version = environ['DIST_VERSION']
with Path(environ['GITHUB_OUTPUT']).open(mode=FILE_APPEND_MODE) as out:
print(f'sdist={project_name}-{version}.tar.gz', file=out)
print(
f'wheel={project_name}-{version}-py3-none-any.whl',
file=out,
)
build:
name: >-
📦 Build dists v${{ needs.pre-setup.outputs.dist-version }}
needs:
- pre-setup
uses: tox-dev/workflow/.github/workflows/reusable-tox.yml@34958348bac6272f9729229f1b5679c76c3f9651 # yamllint disable-line rule:line-length
with:
cache-key-for-dependency-files: >-
${{ needs.pre-setup.outputs.cache-key-for-dep-files }}
check-name: Build dists under 🐍3.12
job-dependencies-context: >- # context for hooks
${{ toJSON(needs) }}
python-version: '3.12'
runner-vm-os: ubuntu-latest
timeout-minutes: 3
toxenv: build-dists
xfail: false
lint:
name: 🧹 Linters${{ '' }} # nest jobs under the same sidebar category
needs:
- build
- pre-setup # transitive, for accessing settings
strategy:
matrix:
runner-vm-os:
- ubuntu-latest
python-version:
- 3.12
toxenv:
- pre-commit
- spellcheck-docs
- build-docs
- doctest-docs
xfail:
- false
fail-fast: false
uses: tox-dev/workflow/.github/workflows/reusable-tox.yml@34958348bac6272f9729229f1b5679c76c3f9651 # yamllint disable-line rule:line-length
with:
cache-key-for-dependency-files: >-
${{ needs.pre-setup.outputs.cache-key-for-dep-files }}
dists-artifact-name: >-
${{ needs.pre-setup.outputs.dists-artifact-name }}
post-toxenv-preparation-command: >-
${{
matrix.toxenv == 'pre-commit'
&& 'python -Im pre_commit install-hooks'
|| ''
}}
python-version: >-
${{ matrix.python-version }}
runner-vm-os: >-
${{ matrix.runner-vm-os }}
# NOTE: `pre-commit --show-diff-on-failure` and
# NOTE: `sphinxcontrib-spellcheck` with Git authors allowlist enabled
# NOTE: both depend on the presence of a Git repository.
source-tarball-name: >-
${{
!contains(
fromJSON('["pre-commit", "spellcheck-docs"]'),
matrix.toxenv
)
&& needs.pre-setup.outputs.sdist-artifact-name
|| ''
}}
# NOTE: `pre-commit` and `sphinxcontrib-spellcheck` both depend on Git.
# NOTE: They may get slower due to network I/O, hence bigger timeout.
timeout-minutes: >-
${{
!contains(
fromJSON('["pre-commit", "spellcheck-docs"]'),
matrix.toxenv
)
&& 4
|| 6
}}
toxenv: >-
${{ matrix.toxenv }}
xfail: >-
${{ matrix.xfail }}
build-wheels-for-tested-arches:
name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category
📦 Build wheels for tested arches${{ '' }}
needs:
- build
- pre-setup # transitive, for accessing settings
strategy:
matrix:
os:
- ubuntu-latest
- windows-latest
- windows-11-arm
- macos-latest
tag:
- ''
- 'musllinux'
exclude:
- os: windows-latest
tag: 'musllinux'
- os: windows-11-arm
tag: 'musllinux'
- os: macos-latest
tag: 'musllinux'
- os: ubuntu-latest
tag: >-
${{
(github.event_name != 'push' || !contains(github.ref, 'refs/tags/'))
&& 'musllinux' || 'none'
}}
uses: ./.github/workflows/reusable-build-wheel.yml
with:
os: ${{ matrix.os }}
tag: ${{ matrix.tag }}
wheel-tags-to-skip: >-
${{
(github.event_name != 'push' || !contains(github.ref, 'refs/tags/'))
&& '*_i686
*-macosx_universal2
*-musllinux_*
*-win32
pp*'
|| (matrix.tag == 'musllinux') && '*-manylinux_* pp*'
|| '*-musllinux_* pp*'
}}
source-tarball-name: >-
${{ needs.pre-setup.outputs.sdist-artifact-name }}
dists-artifact-name: ${{ needs.pre-setup.outputs.dists-artifact-name }}
cython-tracing: >- # Cython line tracing for coverage collection
${{
(
github.event_name == 'push'
&& contains(github.ref, 'refs/tags/')
)
&& 'false'
|| 'true'
}}
metadata-validation:
name: 🧹 Validate metadata of all built dists
needs:
- build # sdist + pure-python wheel
- build-wheels-for-tested-arches # cibuildwheel binary wheels
- pre-setup # transitive, for accessing settings
uses: tox-dev/workflow/.github/workflows/reusable-tox.yml@34958348bac6272f9729229f1b5679c76c3f9651 # yamllint disable-line rule:line-length
with:
cache-key-for-dependency-files: >-
${{ needs.pre-setup.outputs.cache-key-for-dep-files }}
check-name: 🧹 Validate metadata of all built dists
dists-artifact-name: >-
${{ needs.pre-setup.outputs.dists-artifact-name }}
job-dependencies-context: >- # context for hooks
${{ toJSON(needs) }}
python-version: '3.12'
runner-vm-os: ubuntu-latest
source-tarball-name: >-
${{ needs.pre-setup.outputs.sdist-artifact-name }}
timeout-minutes: 3
toxenv: metadata-validation
xfail: false
test:
name: Test
needs:
- build # transitive, ensures dists are built
- build-wheels-for-tested-arches
- pre-setup # transitive, for accessing settings
strategy:
matrix:
pyver:
- 3.14t
- 3.14
# Skipping 3.13t because cibuildwheel dropped support
- 3.13
- 3.12
- 3.11
- >-
3.10
no-extensions: ['', 'Y']
os:
- ubuntu-latest
- macos-latest
- windows-latest
- windows-11-arm
experimental: [false]
exclude:
- os: macos-latest
no-extensions: Y
- os: windows-latest
no-extensions: Y
- os: windows-11-arm
no-extensions: Y
- os: windows-11-arm
pyver: '3.10' # not supported by setup-python action
include:
- pyver: pypy-3.10
no-extensions: Y
experimental: false
os: ubuntu-latest
fail-fast: false
runs-on: ${{ matrix.os }}
timeout-minutes: 5
continue-on-error: ${{ matrix.experimental }}
steps:
- name: Checkout project
uses: actions/checkout@v6
- name: Retrieve the project source from an sdist inside the GHA artifact
uses: re-actors/checkout-python-sdist@release/v2
with:
source-tarball-name: >-
${{ needs.pre-setup.outputs.sdist-artifact-name }}
workflow-artifact-name: >-
${{ needs.pre-setup.outputs.dists-artifact-name }}
- name: Download distributions
uses: actions/download-artifact@v8
with:
path: dist
pattern: ${{ needs.pre-setup.outputs.dists-artifact-name }}*
merge-multiple: true
- name: Setup Python ${{ matrix.pyver }}
id: python-install
uses: astral-sh/setup-uv@v8.1.0
with:
python-version: ${{ matrix.pyver }}
activate-environment: true
# Pre-release interpreters (alpha/beta/rc, `-dev`, or `~`-pinned
# latest) skip the wheel cache so each run resolves freshly
# against whatever the current snapshot ships.
enable-cache: >-
${{
(
startsWith(matrix.pyver, '~')
|| endsWith(matrix.pyver, '-dev')
|| contains(matrix.pyver, 'a')
|| contains(matrix.pyver, 'b')
|| contains(matrix.pyver, 'rc')
)
&& 'false'
|| 'true'
}}
- name: Install dependencies
run: uv pip install -r requirements/codspeed.txt
- name: Install ${{ env.PROJECT_NAME }} from a pre-built wheel
run: >-
uv pip install
--find-links=./dist
--no-index
--no-deps
--force-reinstall
--only-binary=:all:
'${{ env.PROJECT_NAME }}'
- name: Produce the C-files for the Coverage.py Cython plugin
if: >- # Only works if the dists were built with line tracing
!matrix.no-extensions
&& (
github.event_name != 'push'
|| !contains(github.ref, 'refs/tags/')
)
env:
PYTHONPATH: packaging/
run: |
set -eEuo pipefail
uv pip install expandvars
python -m pep517_backend.cli translate-cython
shell: bash
- name: Disable the Cython.Coverage Produce plugin
if: >- # Only works if the dists were built with line tracing
matrix.no-extensions
|| (
github.event_name == 'push'
&& contains(github.ref, 'refs/tags/')
)
run: |
set -eEuo pipefail
sed -i.bak 's/^\s\{2\}Cython\.Coverage$//g' .coveragerc
shell: bash
- name: Run unittests
run: >-
python -Im
pytest
-v
--cov-report xml
--junitxml=.test-results/pytest/test.xml
--${{ matrix.no-extensions == 'Y' && 'no-' || '' }}c-extensions
- name: Produce markdown test summary from JUnit
if: >-
!cancelled()
uses: test-summary/action@v2.6
with:
paths: .test-results/pytest/test.xml
- name: Append coverage results to Job Summary
if: >-
!cancelled()
continue-on-error: true
run: >-
python -Im coverage report --format=markdown
>> "${GITHUB_STEP_SUMMARY}"
shell: bash
- name: Re-run the failing tests with maximum verbosity
if: >-
!cancelled()
&& failure()
run: >- # `exit 1` makes sure that the job remains red with flaky runs
python -Im
pytest
--no-cov
-vvvvv
--lf
-rA
--${{ matrix.no-extensions == 'Y' && 'no-' || '' }}c-extensions
&& exit 1
shell: bash
- name: Send coverage data to Codecov
if: >-
!cancelled()
uses: codecov/codecov-action@v6
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: >-
CI-GHA,
pytest,
OS-${{ runner.os }},
VM-${{ matrix.os }},
Py-${{ steps.python-install.outputs.python-version }}
fail_ci_if_error: false
benchmark:
name: Benchmark
needs:
- build # transitive, ensures dists are built
- build-wheels-for-tested-arches
- pre-setup # transitive, for accessing settings
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout project
uses: actions/checkout@v6
- name: Retrieve the project source from an sdist inside the GHA artifact
uses: re-actors/checkout-python-sdist@release/v2
with:
source-tarball-name: >-
${{ needs.pre-setup.outputs.sdist-artifact-name }}
workflow-artifact-name: >-
${{ needs.pre-setup.outputs.dists-artifact-name }}
- name: Download distributions
uses: actions/download-artifact@v8
with:
path: dist
pattern: ${{ needs.pre-setup.outputs.dists-artifact-name }}*
merge-multiple: true
- name: Setup Python 3.13
id: python-install
uses: astral-sh/setup-uv@v8.1.0
with:
python-version: 3.13
activate-environment: true
enable-cache: true
- name: Install dependencies
run: uv pip install -r requirements/codspeed.txt
- name: Install ${{ env.PROJECT_NAME }} from a pre-built wheel
run: >-
uv pip install
--find-links=./dist
--no-index
--no-deps
--force-reinstall
--only-binary=:all:
'${{ env.PROJECT_NAME }}'
- name: Run benchmarks
uses: CodSpeedHQ/action@v4
with:
token: ${{ secrets.CODSPEED_TOKEN }}
run: python -Im pytest --no-cov -vvvvv --codspeed
mode: instrumentation
test-summary:
name: Test matrix status
if: always()
runs-on: ubuntu-latest
timeout-minutes: 1
needs:
- lint
- metadata-validation
- test
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
pre-deploy:
name: Pre-Deploy
runs-on: ubuntu-latest
timeout-minutes: 1
needs: test-summary
# Run only on pushing a tag
if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
steps:
- name: Dummy
run: |
echo "Predeploy step"
build-wheels-for-odd-archs:
name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category
📦 Build wheels for odd arches${{ '' }}
needs:
- build
- pre-deploy
- pre-setup # transitive, for accessing settings
strategy:
matrix:
qemu:
- aarch64
- ppc64le
- s390x
- armv7l
- riscv64
tag:
- ''
- musllinux
exclude:
- tag: ''
qemu: armv7l
include:
# Build aarch64 natively on the GH-hosted ARM runner instead of
# x86_64-under-QEMU; this drops the build from ~40 min of
# emulation to a few minutes of native execution. Each
# (qemu, tag) combo is listed explicitly so the include
# unambiguously augments the existing matrix cell rather than
# relying on the partial-key merge behaviour of `matrix.include`.
- qemu: aarch64
tag: ''
runner-vm-os: ubuntu-24.04-arm
native-arch: true
- qemu: aarch64
tag: musllinux
runner-vm-os: ubuntu-24.04-arm
native-arch: true
# armv7l also moves to aarch64 hosts. QEMU stays registered so
# binfmt picks up the 32-bit ARM userspace handler regardless of
# whether the host kernel has CONFIG_COMPAT enabled. Even with
# emulation, aarch64-on-aarch64 hosting beats x86_64 by a wide
# margin. armv7l's ``tag: ''`` cell is excluded above, so only
# the musllinux cell is overridden here.
- qemu: armv7l
tag: musllinux
runner-vm-os: ubuntu-24.04-arm
uses: ./.github/workflows/reusable-build-wheel.yml
with:
os: ${{ matrix.runner-vm-os || 'ubuntu-latest' }}
qemu: ${{ matrix.qemu }}
native-arch: ${{ matrix.native-arch && true || false }}
tag: ${{ matrix.tag }}
wheel-tags-to-skip: >-
${{
(matrix.tag == 'musllinux')
&& '*-manylinux_* pp*'
|| '*-musllinux_* pp*'
}}
source-tarball-name: >-
${{ needs.pre-setup.outputs.sdist-artifact-name }}
dists-artifact-name: ${{ needs.pre-setup.outputs.dists-artifact-name }}
deploy:
name: Deploy
needs:
- build
- build-wheels-for-odd-archs
- build-wheels-for-tested-arches
- pre-setup # transitive, for accessing settings
runs-on: ubuntu-latest
timeout-minutes: 14
permissions:
contents: write # IMPORTANT: mandatory for making GitHub Releases
id-token: write # IMPORTANT: mandatory for trusted publishing & sigstore
environment:
name: pypi
url: https://pypi.org/p/${{ env.PROJECT_NAME }}
steps:
- name: Retrieve the project source from an sdist inside the GHA artifact
uses: re-actors/checkout-python-sdist@release/v2
with:
source-tarball-name: >-
${{ needs.pre-setup.outputs.sdist-artifact-name }}
workflow-artifact-name: >-
${{ needs.pre-setup.outputs.dists-artifact-name }}
- name: Download distributions
uses: actions/download-artifact@v8
with:
path: dist
pattern: ${{ needs.pre-setup.outputs.dists-artifact-name }}*
merge-multiple: true
- run: |
tree
- name: Login
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token
- name: Check whether the GitHub Release already exists
# Allows re-running the deploy job after a partial failure (e.g. PyPI
# upload error) without the Make Release step failing with HTTP 422
# because the tag/release was created on a prior attempt. Treat
# only the literal `release not found` reply as "does not exist";
# other failures (auth, rate-limit, network) re-raise so the job
# fails loudly instead of falling through to Make Release.
id: gh-release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ github.ref_name }}
run: |
if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" \
>/dev/null 2>err; then
echo 'exists=true' >> "${GITHUB_OUTPUT}"
elif grep -qx 'release not found' err; then
echo 'exists=false' >> "${GITHUB_OUTPUT}"
else
cat err >&2
exit 1
fi
- name: Make Release
if: steps.gh-release.outputs.exists != 'true'
uses: aio-libs/create-release@v1.6.6
with:
changes_file: CHANGES.rst
version_file: src/${{ env.PROJECT_NAME }}/__init__.py
github_token: ${{ secrets.GITHUB_TOKEN }}
head_line: >-
{version}\n=+\n\n\*\({date}\)\*\n
fix_issue_regex: >-
:issue:`(\d+)`
fix_issue_repl: >-
#\1
- name: >-
Publish 🐍📦 to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
# Allow re-running the deploy job after a partial PyPI upload
# without failing on dists that were already published.
skip-existing: true
- name: Sign the dists with Sigstore
uses: sigstore/gh-action-sigstore-python@v3.3.0
with:
inputs: >-
./dist/${{ needs.pre-setup.outputs.sdist-artifact-name }}
./dist/*.whl
- name: Upload artifact signatures to GitHub Release
# Confusingly, this action also supports updating releases, not
# just creating them. This is what we want here, since we've manually
# created the release above.
uses: softprops/action-gh-release@v3
with:
# dist/ contains the built packages, which smoketest-artifacts/
# contains the signatures and certificates.
files: dist/**
...