feat: cross compile #1947
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI - Aspect Workflows | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: | |
| id-token: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| buildifier: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2 | |
| with: | |
| aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }} | |
| - name: Buildifier | |
| run: aspect buildifier --task-key=buildifier | |
| gazelle: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2 | |
| with: | |
| aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }} | |
| - name: Gazelle | |
| run: aspect gazelle --task-key=gazelle | |
| typecheck: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2 | |
| with: | |
| aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }} | |
| - name: Ruff annotations | |
| run: bazel run @uv -- run --locked --group dev -- ruff check . | |
| - name: Typecheck maintained Python | |
| run: | | |
| runtime310=( | |
| py/private/launcher_env/aspect_rules_py_launcher_env.py | |
| py/private/py_image_layer_validator.py | |
| py/private/py_venv/templates/link.py | |
| py/private/pytest_main.py | |
| py/private/pytest_shard/pytest_shard.py | |
| py/private/unittest_main.py | |
| py/tools/pex/main.py | |
| py/tools/site_merge/site_merge.py | |
| py/tools/unpack/unpack.py | |
| uv/private/gazelle_manifest/tools/generate.py | |
| uv/private/pep517_whl/tools/build_helper.py | |
| uv/private/pep517_whl/tools/memory_monitor.py | |
| uv/private/py_entrypoint_binary/search.py | |
| ) | |
| runtime311=( | |
| uv/private/sdist_configure/detect_native.py | |
| ) | |
| bazel run @uv -- run --locked --group dev -- pyright --pythonversion 3.10 --warnings "${runtime310[@]}" | |
| bazel run @uv -- run --locked --group dev -- pyright --pythonversion 3.11 --warnings "${runtime311[@]}" | |
| bazel run @uv -- run --locked --group dev -- ty check --python-version 3.10 --error-on-warning --output-format github "${runtime310[@]}" | |
| bazel run @uv -- run --locked --group dev -- ty check --python-version 3.11 --error-on-warning --output-format github "${runtime311[@]}" | |
| test: | |
| name: test (${{ matrix.workspace.path }}, ${{ matrix.bazel.id }}) | |
| runs-on: ${{ matrix.workspace.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| workspace: | |
| - { path: ".", slug: "root", runner: "ubuntu-latest" } | |
| - { path: "e2e/cases", slug: "e2e", runner: "ubuntu-22.04-32core" } | |
| - { path: "e2e/crossbuild", slug: "e2e-crossbuild", runner: "ubuntu-22.04-32core" } | |
| - { path: "e2e/interpreter-runtime-metadata", slug: "e2e-interpreter-runtime-metadata", runner: "ubuntu-latest" } | |
| - { path: "e2e/interpreter-toolchain-settings", slug: "e2e-interpreter-toolchain-settings", runner: "ubuntu-latest" } | |
| - { path: "e2e/interpreter-input-validation", slug: "e2e-interpreter-input-validation", runner: "ubuntu-latest" } | |
| - { path: "e2e/interpreter-build-config", slug: "e2e-interpreter-build-config", runner: "ubuntu-latest" } | |
| - { path: "e2e/rules-python-interop", slug: "e2e-rules-python-interop", runner: "ubuntu-latest" } | |
| - { path: "e2e/rules-python-provider-compat", slug: "e2e-rules-python-provider-compat", runner: "ubuntu-latest" } | |
| - { path: "e2e/rules-proto-grpc-python", slug: "e2e-rules-proto-grpc-python", runner: "ubuntu-22.04-32core" } | |
| - { path: "examples/debugger", slug: "examples-debugger", runner: "ubuntu-latest" } | |
| - { path: "examples/dev_deps", slug: "examples-dev_deps", runner: "ubuntu-latest" } | |
| - { path: "examples/django", slug: "examples-django", runner: "ubuntu-latest" } | |
| - { path: "examples/multi_version", slug: "examples-multi_version", runner: "ubuntu-latest" } | |
| - { path: "examples/py_binary", slug: "examples-py_binary", runner: "ubuntu-latest" } | |
| - { path: "examples/py_pex_binary", slug: "examples-py_pex_binary", runner: "ubuntu-latest" } | |
| - { path: "examples/py_venv", slug: "examples-py_venv", runner: "ubuntu-latest" } | |
| - { path: "examples/pytest", slug: "examples-pytest", runner: "ubuntu-latest" } | |
| - { path: "examples/uv_pip_compile", slug: "examples-uv_pip_compile", runner: "ubuntu-latest" } | |
| - { path: "examples/virtual_deps", slug: "examples-virtual_deps", runner: "ubuntu-latest" } | |
| - { path: "examples/protobuf", slug: "examples-protobuf", runner: "ubuntu-22.04-32core" } | |
| bazel: | |
| # An empty version falls back to the workspace's .bazelversion, which validates | |
| # the committed MODULE.bazel.lock; alternate versions may regenerate it instead. | |
| - { id: "bazel-8", version: "", flags: "--bazel-flag=--test_tag_filters=-skip-on-bazel8 --bazel-flag=--lockfile_mode=error" } | |
| - { id: "bazel-9", version: "9.x", flags: "--bazel-flag=--test_tag_filters=-skip-on-bazel9 --bazel-flag=--lockfile_mode=update" } | |
| env: | |
| USE_BAZEL_VERSION: ${{ matrix.bazel.version }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2 | |
| with: | |
| aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }} | |
| # e2e/crossbuild runs an arm64 container from this amd64 runner | |
| - if: matrix.workspace.slug == 'e2e-crossbuild' | |
| uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 | |
| with: | |
| platforms: arm64 | |
| # Run `test //...`, then the workspace's test.sh if it has one | |
| - name: Test | |
| working-directory: ${{ matrix.workspace.path }} | |
| run: | | |
| aspect test --task-key=test-${{ matrix.workspace.slug }}-${{ matrix.bazel.id }} ${{ matrix.bazel.flags }} -- //... | |
| if [ -f test.sh ]; then | |
| bash test.sh | |
| fi | |
| # macOS minutes are billed at 10X, so it only covers the root and e2e consumer | |
| # workspaces, on main plus branches whose name contains 'macos'. | |
| # https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#included-storage-and-minutes | |
| crossbuild-images-darwin: | |
| name: build on darwin | |
| if: >- | |
| ${{ github.ref_name == 'main' | |
| || contains(github.head_ref || github.ref_name, 'macos') }} | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0 | |
| with: | |
| bazelisk-cache: true | |
| disk-cache: macos-crossbuild-images | |
| repository-cache: true | |
| - name: Cross-build linux image tarballs from macOS | |
| working-directory: e2e/crossbuild | |
| shell: bash | |
| run: | | |
| mkdir -p "${RUNNER_TEMP}/images" | |
| for case in psutil contourpy rpds_py; do | |
| for arch in amd64 arm64; do | |
| bazel build --config=ci --remote_download_outputs=toplevel --action_env=MESON_FORCE_BACKTRACE=1 "//${case}:${arch}_tarball" | |
| cp "bazel-bin/${case}/${arch}_load/tarball.tar" "${RUNNER_TEMP}/images/${case}-${arch}.tar" | |
| done | |
| done | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: crossbuild-images-darwin | |
| path: ${{ runner.temp }}/images/*.tar | |
| retention-days: 1 | |
| crossbuild-run-darwin-built: | |
| name: run on linux ${{ matrix.arch }} | |
| needs: crossbuild-images-darwin | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - { runner: "ubuntu-latest", arch: "amd64" } | |
| - { runner: "ubuntu-24.04-arm", arch: "arm64" } | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: crossbuild-images-darwin | |
| path: images | |
| - name: Run darwin-built images on native ${{ matrix.arch }} | |
| run: | | |
| status=0 | |
| for tar in images/*-${{ matrix.arch }}.tar; do | |
| tag=$(docker load -i "${tar}" | sed -n 's/^Loaded image: //p') | |
| echo "::group::${tag}" | |
| docker run --rm "${tag}" || status=1 | |
| echo "::endgroup::" | |
| done | |
| exit "${status}" | |
| crossbuild-images-amd64: | |
| name: build on amd | |
| runs-on: ubuntu-22.04-32core | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2 | |
| with: | |
| aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }} | |
| - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 | |
| with: | |
| platforms: arm64 | |
| - name: Cross-build arm64 image tarballs | |
| working-directory: e2e/crossbuild | |
| run: | | |
| mkdir -p "${RUNNER_TEMP}/images" | |
| targets=$(bazel query 'filter(".*:arm64_tarball$", //...)') | |
| bazel build --remote_download_outputs=toplevel ${targets} | |
| for target in ${targets}; do | |
| case="${target#//}" | |
| case="${case%:arm64_tarball}" | |
| cp "bazel-bin/${case}/arm64_load/tarball.tar" "${RUNNER_TEMP}/images/${case}-arm64.tar" | |
| done | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: crossbuild-images-amd64-built | |
| path: ${{ runner.temp }}/images/*.tar | |
| retention-days: 1 | |
| crossbuild-run-amd64-built: | |
| name: run on arm | |
| needs: crossbuild-images-amd64 | |
| runs-on: ubuntu-24.04-arm | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: crossbuild-images-amd64-built | |
| path: images | |
| - name: Run amd64-built images on native arm64 | |
| run: | | |
| status=0 | |
| for tar in images/*-arm64.tar; do | |
| tag=$(docker load -i "${tar}" | sed -n 's/^Loaded image: //p') | |
| echo "::group::${tag}" | |
| docker run --rm "${tag}" || status=1 | |
| echo "::endgroup::" | |
| done | |
| exit "${status}" | |
| crossbuild-images-arm64: | |
| name: build on arm | |
| runs-on: ubuntu-24.04-arm | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: aspect-build/setup-aspect@c22a8f64fb38f82f59ce809cd7eb9f8ae096da44 # v2026.23.2 | |
| with: | |
| aspect-api-token: ${{ secrets.ASPECT_API_TOKEN }} | |
| - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 | |
| with: | |
| platforms: amd64 | |
| - name: Cross-build amd64 image tarballs from arm64 | |
| working-directory: e2e/crossbuild | |
| run: | | |
| mkdir -p "${RUNNER_TEMP}/images" | |
| for case in psutil contourpy rpds_py; do | |
| # toplevel: build-without-the-bytes must still materialize | |
| # the tarball this job exists to upload. | |
| bazel build --config=ci --remote_download_outputs=toplevel "//${case}:amd64_tarball" | |
| cp "bazel-bin/${case}/amd64_load/tarball.tar" "${RUNNER_TEMP}/images/${case}-amd64.tar" | |
| done | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: crossbuild-images-arm-built | |
| path: ${{ runner.temp }}/images/*.tar | |
| retention-days: 1 | |
| crossbuild-run-arm64-built: | |
| name: run on amd | |
| needs: crossbuild-images-arm64 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: crossbuild-images-arm-built | |
| path: images | |
| - name: Run arm-built images on native amd64 | |
| run: | | |
| status=0 | |
| for tar in images/*-amd64.tar; do | |
| tag=$(docker load -i "${tar}" | sed -n 's/^Loaded image: //p') | |
| echo "::group::${tag}" | |
| docker run --rm "${tag}" || status=1 | |
| echo "::endgroup::" | |
| done | |
| exit "${status}" | |
| test-all: | |
| name: test-all | |
| runs-on: ubuntu-latest | |
| needs: [buildifier, gazelle, typecheck, crossbuild-run-amd64-built, crossbuild-run-arm64-built, crossbuild-run-darwin-built] | |
| if: always() | |
| steps: | |
| - name: All required jobs succeeded | |
| run: | | |
| for result in \ | |
| "typecheck=${{ needs.typecheck.result }}" \ | |
| "buildifier=${{ needs.buildifier.result }}" \ | |
| "gazelle=${{ needs.gazelle.result }}" \ | |
| "crossbuild-run-amd64-built=${{ needs['crossbuild-run-amd64-built'].result }}" \ | |
| "crossbuild-run-arm64-built=${{ needs['crossbuild-run-arm64-built'].result }}"; do | |
| if [ "${result#*=}" != "success" ]; then | |
| echo "Required job did not succeed: ${result}" | |
| exit 1 | |
| fi | |
| done | |
| # The darwin pipeline is gated to main and '*macos*' branches: | |
| # skipped is expected elsewhere; anything that ran must pass. | |
| case "${{ needs['crossbuild-run-darwin-built'].result }}" in | |
| success|skipped) ;; | |
| *) | |
| echo "crossbuild-run-darwin-built did not succeed: ${{ needs['crossbuild-run-darwin-built'].result }}" | |
| exit 1 | |
| ;; | |
| esac | |