add workflow to measure execute time (#789)

* add workflow for linux * add hyperfine and comments * add runs * fix run * fix run * fix run * fix run * fix command * show errors * change how handles project building * typo in command * forgot issue-comment-id in update comment * better result printing * change approach * versions fix * build binaries correctly * require binaries to run benches * typo * remove unwanted programs * fix run command * checkout base before running * checkout base before running * fix run hyperfine command * fix run hyperfine command * typos * fix OUTPUT_DIR env var * fix ROOT_DIR var * forgot to make the output's dir * fix * fix hyperfine command * fix * change ouput's dir * change ouput's dir * fix * check with one branch * check with other branch * remove checkout * use checkout before running bench * use onw branh * create cache againt * clean unwanted dir * check errors * change command * remove unwanted dir * cache program to run * fix caching * fix caching * forgot to add env var * typo * typo * typo * checkout first * revert * on pull_request all branches * change from base to head * show error * make deps * run both branches * run both branches * typo * make corrections * remove unwanted dirs * fix bench command * fix bench command * add warmup * add warmup * change hyperfine command * showoutput * add runtime library * cache hits * fix typo * fix typo * remove conditional temporarily * typos * debug * debug * debug * debug * typo * enable shell * debug * debug * debug * debug * typo * typo * typo * change cache files name * cache everything again * cache everything again * debug * debug * debug * debug * debug * debug * add head variant * add head variant * add head variant * add more programs * add more programs * add more programs * typo * better title * better title
lambdaclass · Sep 19, 2024 · 7faf271 · 7faf271
1 parent 0eb9d45
commit 7faf271
Show file tree

Hide file tree

Showing 2 changed files with 168 additions and 0 deletions.
diff --git a/.github/workflows/bench-hyperfine.yml b/.github/workflows/bench-hyperfine.yml
@@ -86,3 +86,169 @@ jobs:
           issue-number: ${{ github.event.pull_request.number }}
           body-path: bench-hyperfine.md
           edit-mode: replace
+
+  build-binaries:
+    env:
+      MLIR_SYS_180_PREFIX: /usr/lib/llvm-18/
+      LLVM_SYS_181_PREFIX: /usr/lib/llvm-18/
+      TABLEGEN_180_PREFIX: /usr/lib/llvm-18/
+      RUST_LOG: cairo_native=debug,cairo_native_test=debug
+    strategy:
+      matrix:
+        branch: [ base, head ]
+    name: Build cairo-native-run for ${{ matrix.branch }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cache binary
+        uses: actions/cache@v3
+        id: cache-binary 
+        with:
+          path: bin/cairo-native-run-${{ matrix.branch }}
+          key: binary-${{ github.event.pull_request[matrix.branch].sha }}
+
+      - name: Cache runtime-library
+        uses: actions/cache@v3
+        id: cache-library 
+        with:
+          path: lib/libcairo-native-runtime-${{ matrix.branch }}.a
+          key: library-${{ github.event.pull_request[matrix.branch].sha }} 
+
+      - name: check and free hdd space left
+        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' }}
+        run: |
+          echo "Listing 20 largest packages"
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
+          df -h
+          sudo apt-get update
+          sudo apt-get remove -y '^llvm-.*'
+          sudo apt-get remove -y 'php.*'
+          sudo apt-get remove -y '^dotnet-.*'
+          sudo apt-get remove -y '^temurin-.*'
+          sudo apt-get remove -y azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox powershell mono-devel
+          sudo apt-get autoremove -y
+          sudo apt-get clean
+          df -h
+          echo "Removing large directories"
+          # deleting 15GB
+          sudo rm -rf /usr/share/dotnet/
+          sudo rm -rf /usr/local/lib/android
+          df -h
+
+      - name: Install Rust
+        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
+        uses: dtolnay/[email protected]
+
+      - name: add llvm deb repository
+        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
+        uses: myci-actions/add-deb-repo@11
+        with:
+          repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
+          repo-name: llvm-repo
+          keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
+
+      - name: Install LLVM
+        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
+        run: sudo apt-get update && sudo apt-get install llvm-18 llvm-18-dev llvm-18-runtime clang-18 clang-tools-18 lld-18 libpolly-18-dev libmlir-18-dev mlir-18-tools
+
+      - name: Checkout base commit
+        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request[matrix.branch].sha }}
+
+      - name: Fetch Rust cache
+        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
+        uses: Swatinem/rust-cache@v2
+
+      - name: Build binary
+        if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }}
+        run: |
+          make build
+          mkdir lib bin
+          cp target/release/cairo-native-run bin/cairo-native-run-${{ matrix.branch }}
+          cp target/release/libcairo_native_runtime.a lib/libcairo-native-runtime-${{ matrix.branch }}.a
+  
+  hyperfine-prs:
+    name: Bench PR (linux, amd64)
+    needs: [ build-binaries ]
+    runs-on: ubuntu-latest
+    env:
+      PROGRAM: fib_2M
+      OUTPUT_DIR: bench-outputs
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Hyperfine
+        uses: taiki-e/install-action@v2
+        with:
+          tool: [email protected]
+
+      - name: Fetch base binary
+        uses: actions/cache/restore@v3
+        with:
+          path: bin/cairo-native-run-base
+          key: binary-${{ github.event.pull_request.base.sha }}
+
+      - name: Fetch HEAD binary
+        uses: actions/cache/restore@v3
+        with:
+          path: bin/cairo-native-run-head
+          key: binary-${{ github.event.pull_request.head.sha }}
+
+      - name: Fetch base runtime-library
+        uses: actions/cache/restore@v3
+        with:
+          path: lib/libcairo-native-runtime-base.a
+          key: library-${{ github.event.pull_request.base.sha }}
+
+      - name: Fetch head runtime-library
+        uses: actions/cache/restore@v3
+        with:
+          path: lib/libcairo-native-runtime-head.a
+          key: library-${{ github.event.pull_request.head.sha }}
+
+      - name: Build Dependencies
+        run: make deps
+
+      - name: Run Program
+        run: |
+          mkdir $OUTPUT_DIR
+          cp programs/benches/*.cairo $OUTPUT_DIR
+          for f in $OUTPUT_DIR/*.cairo; do
+            hyperfine \
+            --export-markdown "${f%.*}.md" \
+            --export-json "${f%.*}.json" \
+            --warmup 3 \
+            -n "head $(basename $f) (JIT)" "./bin/cairo-native-run-head --run-mode=jit -s $f --opt-level 3 --available-gas 18446744073709551615" \
+            -n "base $(basename $f) (JIT)" "./bin/cairo-native-run-base --run-mode=jit -s $f --opt-level 3 --available-gas 18446744073709551615" \
+            -n "head $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-head.a ./bin/cairo-native-run-head --run-mode=aot -s $f --opt-level 3 --available-gas 18446744073709551615" \
+            -n "base $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-base.a ./bin/cairo-native-run-base --run-mode=aot -s $f --opt-level 3 --available-gas 18446744073709551615" \
+            >> /dev/stderr
+          done
+      - name: Print tables
+        run: |
+          {
+            echo "Benchmark results Main vs HEAD"
+            for f in $OUTPUT_DIR/*.md; do
+              echo
+              cat $f
+            done
+          } | tee -a comment_body.md
+
+      - name: Find Bench Comment
+        continue-on-error: true
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: Benchmarking
+
+      - name: Create or update bench comment
+        continue-on-error: true
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: comment_body.md
+          edit-mode: replace
diff --git a/.gitignore b/.gitignore
@@ -34,3 +34,5 @@ cairo-*.tar
 
 *.ipynb
 cairo-native-stress-logs.jsonl
+
+.DS_Store
Original file line number	Diff line number	Diff line change
Expand Up		@@ -34,3 +34,5 @@ cairo-*.tar

		*.ipynb
		cairo-native-stress-logs.jsonl

		.DS_Store