bench-hyperfine.yml

name: Bench

on:
  pull_request:
    branches: [main]
  merge_group:
    types: [checks_requested]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always

jobs:
  bench-hyperfine:
    name: Hyperfine
    runs-on: ubuntu-24.04
    env:
      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
      MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
      TABLEGEN_190_PREFIX: /usr/lib/llvm-19/
      LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
    steps:
      - uses: actions/checkout@v4
      - name: check and free hdd space left
        run: |
          echo "Listing 20 largest packages"
          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
          df -h
          sudo apt-get update
          sudo apt-get remove -y '^llvm-.*'
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^dotnet-.*'
          sudo apt-get remove -y '^temurin-.*'
          sudo apt-get remove -y azure-cli microsoft-edge-stable google-chrome-stable firefox mono-devel
          sudo apt-get autoremove -y
          sudo apt-get clean
          df -h
          echo "Removing large directories"
          # deleting 15GB
          sudo rm -rf /usr/share/dotnet/
          sudo rm -rf /usr/local/lib/android
          df -h
      - uses: dtolnay/rust-toolchain@1.82.0
        with:
          components: clippy
      - uses: Swatinem/rust-cache@v2
      - name: add llvm deb repository
        uses: myci-actions/add-deb-repo@11
        with:
          repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main
          repo-name: llvm-repo
          keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
      - name: Install LLVM
        run: sudo apt-get install llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools
      - name: Install hyperfine
        uses: taiki-e/install-action@v2
        with:
          tool: hyperfine@1.16
      - name: Install deps
        run: make deps
      - name: Build project
        run: make build
      - name: Build runtime subproject
        run: make runtime-ci
      - name: Run benchmarks
        run: ./scripts/bench-hyperfine.sh programs/benches/*.cairo
      - name: Create markdown file
        run: bash .github/scripts/merge-benches.sh

      - name: Find Bench Comment
        continue-on-error: true
        uses: peter-evans/find-comment@v3
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: "github-actions[bot]"
          body-includes: Benchmarking
      - name: Create or update bench comment
        continue-on-error: true
        uses: peter-evans/create-or-update-comment@v4
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body-path: bench-hyperfine.md
          edit-mode: replace

  build-binaries:
    env:
      MLIR_SYS_190_PREFIX: /usr/lib/llvm-19/
      LLVM_SYS_191_PREFIX: /usr/lib/llvm-19/
      TABLEGEN_190_PREFIX: /usr/lib/llvm-19/
      RUST_LOG: cairo_native=debug,cairo_native_test=debug
    strategy:
      matrix:
        branch: [ base, head ]
    name: Build cairo-native-run for ${{ matrix.branch }}
    runs-on: ubuntu-24.04
    steps:
      - name: Cache binary
        uses: actions/cache@v3
        id: cache-binary
        with:
          path: bin/cairo-native-run-${{ matrix.branch }}
          key: binary-${{ github.event.pull_request[matrix.branch].sha }}

      - name: Cache runtime-library
        uses: actions/cache@v3
        id: cache-library
        with:
          path: lib/libcairo-native-runtime-${{ matrix.branch }}.a
          key: library-${{ github.event.pull_request[matrix.branch].sha }}

      - name: check and free hdd space left
        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' }}
        run: |
          echo "Listing 20 largest packages"
          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
          df -h
          sudo apt-get update
          sudo apt-get remove -y '^llvm-.*'
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^dotnet-.*'
          sudo apt-get remove -y '^temurin-.*'
          sudo apt-get remove -y azure-cli microsoft-edge-stable google-chrome-stable firefox mono-devel
          sudo apt-get autoremove -y
          sudo apt-get clean
          df -h
          echo "Removing large directories"
          # deleting 15GB
          sudo rm -rf /usr/share/dotnet/
          sudo rm -rf /usr/local/lib/android
          df -h

      - name: Install Rust
        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
        uses: dtolnay/rust-toolchain@1.82.0

      - name: add llvm deb repository
        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
        uses: myci-actions/add-deb-repo@11
        with:
          repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main
          repo-name: llvm-repo
          keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key

      - name: Install LLVM
        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
        run: sudo apt-get update && sudo apt-get install llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools

      - name: Checkout base commit
        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request[matrix.branch].sha }}

      - name: Fetch Rust cache
        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
        uses: Swatinem/rust-cache@v2

      - name: Build binary
        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
        run: |
          make build
          mkdir lib bin
          cp target/release/cairo-native-run bin/cairo-native-run-${{ matrix.branch }}
          cp target/release/libcairo_native_runtime.a lib/libcairo-native-runtime-${{ matrix.branch }}.a

  hyperfine-prs:
    name: Bench PR (linux, amd64)
    needs: [ build-binaries ]
    runs-on: ubuntu-24.04
    env:
      PROGRAM: fib_2M
      OUTPUT_DIR: bench-outputs
    steps:
      - uses: actions/checkout@v4

      - name: Install Hyperfine
        uses: taiki-e/install-action@v2
        with:
          tool: hyperfine@1.16

      - name: Fetch base binary
        uses: actions/cache/restore@v3
        with:
          path: bin/cairo-native-run-base
          key: binary-${{ github.event.pull_request.base.sha }}

      - name: Fetch HEAD binary
        uses: actions/cache/restore@v3
        with:
          path: bin/cairo-native-run-head
          key: binary-${{ github.event.pull_request.head.sha }}

      - name: Fetch base runtime-library
        uses: actions/cache/restore@v3
        with:
          path: lib/libcairo-native-runtime-base.a
          key: library-${{ github.event.pull_request.base.sha }}

      - name: Fetch head runtime-library
        uses: actions/cache/restore@v3
        with:
          path: lib/libcairo-native-runtime-head.a
          key: library-${{ github.event.pull_request.head.sha }}

      - name: Build Dependencies
        run: make deps

      - name: Run Program
        run: |
          mkdir $OUTPUT_DIR
          cp programs/benches/*.cairo $OUTPUT_DIR
          for f in $OUTPUT_DIR/*.cairo; do
            hyperfine \
            --export-markdown "${f%.*}.md" \
            --export-json "${f%.*}.json" \
            --warmup 3 \
            -n "head $(basename $f) (JIT)" "./bin/cairo-native-run-head --run-mode=jit -s $f --opt-level 3 --available-gas 19446744073709551615" \
            -n "base $(basename $f) (JIT)" "./bin/cairo-native-run-base --run-mode=jit -s $f --opt-level 3 --available-gas 19446744073709551615" \
            -n "head $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-head.a ./bin/cairo-native-run-head --run-mode=aot -s $f --opt-level 3 --available-gas 19446744073709551615" \
            -n "base $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-base.a ./bin/cairo-native-run-base --run-mode=aot -s $f --opt-level 3 --available-gas 19446744073709551615" \
            >> /dev/stderr
          done
      - name: Print tables
        run: |
          {
            echo "${bold}Benchmark results Main vs HEAD.${normal}"
            for f in $OUTPUT_DIR/*.md; do
              echo
              cat $f
            done
          } | tee -a comment_body.md

      - name: Find Bench Comment
        continue-on-error: true
        uses: peter-evans/find-comment@v3
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: "github-actions[bot]"
          body-includes: Benchmark results Main vs HEAD

      - name: Create or update bench comment
        continue-on-error: true
        uses: peter-evans/create-or-update-comment@v4
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body-path: comment_body.md
          edit-mode: replace