ci: add continuous benchmarking

.github/benchmarks/general.json

@@ -0,0 +1,41 @@
+[
+    {
+       "name": "1 core",
+       "bin": "startup_benchmark",
+       "manifest_path": "Cargo.toml",
+       "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark",
+       "external_time": true,
+       "iterations": 25,
+       "group": "General",
+       "plot_group": "Startup Time"
+    },
+    {
+        "name": "2 cores",
+        "bin": "startup_benchmark",
+        "manifest_path": "Cargo.toml",
+        "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark",
+        "external_time": true,
+        "iterations": 25,
+        "group": "General",
+        "plot_group": "Startup Time"
+     },
+    {
+        "name": "4 cores",
+        "bin": "startup_benchmark",
+        "manifest_path": "Cargo.toml",
+        "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark",
+        "external_time": true,
+        "iterations": 25,
+        "group": "General",
+        "plot_group": "Startup Time"
+    },
+    {
+        "name": "Pi Multithreaded Benchmark",
+        "bin": "multithreaded_benchmark",
+        "manifest_path": "Cargo.toml",
+        "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark",
+        "iterations": 25,
+        "group": "General",
+        "plot_group": "Multithreaded Pi Efficiency"
+    }
+]

.github/benchmarks/misc.json

@@ -0,0 +1,26 @@
+[
+    {
+        "name": "Micro Benchmarks",
+        "bin": "micro_benchmarks",
+        "manifest_path": "benches/micro/Cargo.toml",
+        "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks",
+        "iterations": 25,
+        "group": "Micro"
+    },
+    {
+        "name": "Allocation Benchmarks",
+        "bin": "alloc_benchmarks",
+        "manifest_path": "benches/alloc/Cargo.toml",
+        "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 5G -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks",
+        "iterations": 50,
+        "group": "Allocations"
+    },
+    {
+        "name": "Mutex Stress Test",
+        "bin": "mutex",
+        "manifest_path": "benches/mutex/Cargo.toml",
+        "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex",
+        "iterations": 50,
+        "group": "Mutex"
+    }
+]

.github/workflows/benchmark.yml

@@ -0,0 +1,91 @@
+name: Benchmark
+
+on:
+  pull_request_target:
+  merge_group:
+  push:
+    branches:
+      - main
+
+permissions:
+          pull-requests: write
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+    benchmark:
+        name: Run Benchmarks
+        runs-on: [self-hosted]
+        environment: ${{ github.event_name == 'push' && 'benchmark' || '' }}
+
+        strategy:
+          matrix:
+            include:
+              - benchmark-file: /kernel/.github/benchmarks/general.json
+                benchmark-matrix-name: General
+                benchmark-build: true
+              - benchmark-file: /kernel/.github/benchmarks/misc.json
+                benchmark-matrix-name: Misc
+                benchmark-build: false
+
+        steps:
+          - name: Checkout hermit-rs
+            uses: actions/checkout@v4
+            with:
+              repository: hermit-os/hermit-rs
+              submodules: true
+          - name: Remove hermit-kernel submodule
+            run: git rm -r kernel
+          - name: Checkout hermit-kernel
+            uses: actions/checkout@v4
+            with:
+              path: kernel
+          - name: Install dependencies
+            run: |
+              sudo apt-get update
+              sudo apt-get install -y --no-install-recommends qemu-system-x86 cpu-checker parallel gh
+          - name: Download loader
+            run: gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64
+          - uses: mkroening/rust-toolchain-toml@main
+          - name: Install uhyve
+            run: cargo +stable install --locked uhyve
+          - name: Check KVM availability
+            run: |
+              lscpu
+              kvm-ok
+          - uses: dtolnay/rust-toolchain@stable
+          - uses: dtolnay/rust-toolchain@nightly
+            with:
+              components: rust-src
+          - uses: mkroening/rust-toolchain-toml@main
+          - run: rustup component add llvm-tools
+            working-directory: .
+          - run: rustup target add wasm32-wasip1
+            working-directory: .
+          - uses: mkroening/rust-toolchain-toml@main
+            with:
+              toolchain-file: "kernel/rust-toolchain.toml"
+          - name: Set up tap device
+            if: matrix.benchmark-matrix-name == 'Netbench'
+            run: |
+              sudo ip tuntap add tap10 mode tap
+              sudo ip addr add 10.0.5.1/24 broadcast 10.0.5.255 dev tap10
+              sudo ip link set dev tap10 
+              echo 1 | sudo tee /proc/sys/net/ipv4/conf/tap10/proxy_arp
+          - name: Run benchmarks
+            uses: hermit-os/hermit-bench@main
+            id: run-bench
+            with: 
+              benchmark-file: ${{ matrix.benchmark-file }}
+          - name: Store benchmark results
+            uses: hermit-os/github-action-benchmark@main
+            with:
+              tool: 'hermit-bench'
+              output-file-path: ${{ steps.run-bench.outputs.result-file }}
+              github-token: ${{ github.event_name == 'push' && secrets.HERMIT_BENCH_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+              benchmark-data-dir-path: benchmarks
+              gh-repository: github.com/hermit-os/hermit-bench
+              comment-always: true
+              benchmark-matrix-name: ${{ matrix.benchmark-matrix-name }}
+              auto-push: ${{ github.event_name == 'push' }} 

.github/workflows/publish_docs.yml

@@ -38,5 +38,6 @@ jobs:
         with:
           target_branch: gh-pages
           build_dir: target/x86_64-unknown-none/doc
+          keep_history: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}