From 292231e10aaf2fb2fa8939a85764b7549f91c9c2 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Tue, 20 May 2025 15:39:43 +0200 Subject: [PATCH 01/17] ci: add continuous benchmarking --- .github/benchmarks/bench.json | 127 ++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 55 ++++++++++++++- 2 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 .github/benchmarks/bench.json diff --git a/.github/benchmarks/bench.json b/.github/benchmarks/bench.json new file mode 100644 index 0000000000..2915ff9914 --- /dev/null +++ b/.github/benchmarks/bench.json @@ -0,0 +1,127 @@ +[ + { + "name": "1 core", + "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "2 cores", + "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "4 cores", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "alloc_benchmarks", + "path": "target/x86_64-unknown-hermit/release/alloc_benchmarks", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "micro_benchmarks", + "path": "target/x86_64-unknown-hermit/release/micro_benchmarks", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "mutex test", + "path": "target/x86_64-unknown-hermit/release/mutex", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "Micro Benchmarks", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks", + "iterations": 20, + "group": "Micro" + }, + { + "name": "Allocation Benchmarks", + "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 5G -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks", + "iterations": 30, + "group": "Allocations" + }, + { + "name": "Mutex Stress Test", + "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex", + "iterations": 40, + "group": "Mutex" + }, + { + "name": "Pi Multithreaded Benchmark", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark", + "iterations": 15, + "group": "General", + "plot_group": "Multithreaded Pi Efficiency" + }, + { + "name": "Netbench TCP BW - Client", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 1000\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Bandwidth - 1000x1MB" + }, + { + "name": "Netbench TCP BW - Server", + "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 1000\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Bandwidth - 1000x1MB" + }, + { + "name": "Netbench TCP Latency - Client", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 250' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 250\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Latency - 250x1MB" + }, + { + "name": "Netbench TCP Latency - Server", + "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 250\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Latency - 250x1MB" + }, + { + "name": "Netbench UDP BW - Server", + "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::7878-:7878,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000 \"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Bandwidth - 1000x1472B" + }, + { + + "name": "Netbench UDP BW - Client", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000 \"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Bandwidth - 1000x1472B" + }, + { + + "name": "Netbench UDP Latency - Client", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Latency - 250x1472B" + }, + { + "name": "Netbench UDP Latency - Server", + "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::7878-:7878,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000 \"'", + "iterations": 3, + "group": "Networking", + "plot_group": "Netbench Latency - 250x1472B" + } +] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ee3544a35..3303d91425 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: pull_request: merge_group: - + env: GH_TOKEN: ${{ github.token }} RUSTFLAGS: -Dwarnings @@ -249,3 +249,56 @@ jobs: if: matrix.arch == 'x86_64' - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} --package thread_test --smp 4 qemu ${{ matrix.flags }} if: matrix.arch == 'x86_64' + + benchmark: + name: Bench + runs-on: [self-hosted] + steps: + - name: Checkout hermit-rs + uses: actions/checkout@v4 + with: + repository: hermit-os/hermit-rs + submodules: true + - name: Remove hermit-kernel submodule + run: git rm -r kernel + working-directory: . + - name: Checkout hermit-kernel + uses: actions/checkout@v4 + with: + path: kernel + - name: Install QEMU + run: | + sudo apt-get update + sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm + - uses: mkroening/rust-toolchain-toml@main + - name: Download loader + run: | + sudo apt install gh -y + gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64 + - name: Install uhyve + run: cargo +stable install --locked uhyve + - name: Check KVM availability + shell: bash + run: | + lscpu + kvm-ok + - name: Add wasm target + run: | + rustup component add llvm-tools + rustup target add wasm32-wasip1 + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - name: Run benchmarks + uses: hermit-os/hermit-bench@kernel + with: + benchmark-file: /kernel/.github/benchmarks/bench.json + build-command: "HERMIT_LOG_LEVEL_FILTER=Off cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" + - name: Store benchmark result + # Only save the result if the commit is on main + # This is to avoid saving results for PRs + if: (github.event_name == 'push' && github.ref == 'refs/heads/main') + uses: hermit-os/github-action-benchmark@hermit-rs + with: + tool: 'hermit-bench' + output-file-path: results.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true From 195b9b158318c91e59c5f17fcb94b9f33d32a625 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Wed, 21 May 2025 19:41:26 +0200 Subject: [PATCH 02/17] fix(cb): install parallel for networking benchmarks --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3303d91425..387552d30e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: pull_request: merge_group: + env: GH_TOKEN: ${{ github.token }} RUSTFLAGS: -Dwarnings @@ -271,6 +272,8 @@ jobs: sudo apt-get update sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm - uses: mkroening/rust-toolchain-toml@main + - name: Install parallel + run: sudo apt install parallel -y - name: Download loader run: | sudo apt install gh -y From 17609ddf86f8d77e634425ace60c6a99dbf30386 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Mon, 26 May 2025 11:47:10 +0200 Subject: [PATCH 03/17] feat(bench): Parallel execution of benchmarks --- .github/benchmarks/general.json | 45 +++++++ .github/benchmarks/misc.json | 38 ++++++ .../benchmarks/{bench.json => netbench.json} | 77 +++++------- .github/workflows/ci.yml | 118 ++++++++++-------- 4 files changed, 178 insertions(+), 100 deletions(-) create mode 100644 .github/benchmarks/general.json create mode 100644 .github/benchmarks/misc.json rename .github/benchmarks/{bench.json => netbench.json} (69%) diff --git a/.github/benchmarks/general.json b/.github/benchmarks/general.json new file mode 100644 index 0000000000..8acc186828 --- /dev/null +++ b/.github/benchmarks/general.json @@ -0,0 +1,45 @@ +[ + { + "name": "1 core", + "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "2 cores", + "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "4 cores", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "external_time": true, + "iterations": 25, + "group": "General", + "plot_group": "Startup Time" + }, + { + "name": "Pi Multithreaded Benchmark", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark", + "iterations": 15, + "group": "General", + "plot_group": "Multithreaded Pi Efficiency" + }, + { + "name": "Pi Multithreaded Benchmark", + "path": "target/x86_64-unknown-hermit/release/multithreaded_benchmark", + "group": "File Size", + "plot_group": "General Benchmark File Size" + }, + { + "name": "Startup Benchmark", + "path": "target/x86_64-unknown-hermit/release/startup_benchmark", + "group": "File Size", + "plot_group": "General Benchmark File Size" + } +] diff --git a/.github/benchmarks/misc.json b/.github/benchmarks/misc.json new file mode 100644 index 0000000000..cfa64bb7f1 --- /dev/null +++ b/.github/benchmarks/misc.json @@ -0,0 +1,38 @@ +[ + { + "name": "alloc_benchmarks", + "path": "target/x86_64-unknown-hermit/release/alloc_benchmarks", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "micro_benchmarks", + "path": "target/x86_64-unknown-hermit/release/micro_benchmarks", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "mutex test", + "path": "target/x86_64-unknown-hermit/release/mutex", + "group": "File Size", + "plot_group": "Benchmark File Size" + }, + { + "name": "Micro Benchmarks", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks", + "iterations": 20, + "group": "Micro" + }, + { + "name": "Allocation Benchmarks", + "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 5G -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks", + "iterations": 30, + "group": "Allocations" + }, + { + "name": "Mutex Stress Test", + "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex", + "iterations": 40, + "group": "Mutex" + } +] diff --git a/.github/benchmarks/bench.json b/.github/benchmarks/netbench.json similarity index 69% rename from .github/benchmarks/bench.json rename to .github/benchmarks/netbench.json index 2915ff9914..3214c59cee 100644 --- a/.github/benchmarks/bench.json +++ b/.github/benchmarks/netbench.json @@ -1,70 +1,51 @@ [ { - "name": "1 core", - "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", - "external_time": true, - "iterations": 25, - "group": "General", - "plot_group": "Startup Time" - }, - { - "name": "2 cores", - "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", - "external_time": true, - "iterations": 25, - "group": "General", - "plot_group": "Startup Time" - }, - { - "name": "4 cores", - "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", - "external_time": true, - "iterations": 25, - "group": "General", - "plot_group": "Startup Time" + "name": "Netbench TCP BW - Client", + "path": "target/x86_64-unknown-hermit/release/tcp-client-bw", + "group": "File Size", + "plot_group": "Netbench Benchmark File Size" }, { - "name": "alloc_benchmarks", - "path": "target/x86_64-unknown-hermit/release/alloc_benchmarks", + "name": "Netbench TCP BW - Server", + "path": "target/x86_64-unknown-hermit/release/tcp-server-bw", "group": "File Size", - "plot_group": "Benchmark File Size" + "plot_group": "Netbench Benchmark File Size" }, { - "name": "micro_benchmarks", - "path": "target/x86_64-unknown-hermit/release/micro_benchmarks", + "name": "Netbench UDP BW - Client", + "path": "target/x86_64-unknown-hermit/release/udp-client-bw", "group": "File Size", - "plot_group": "Benchmark File Size" + "plot_group": "Netbench Benchmark File Size" }, { - "name": "mutex test", - "path": "target/x86_64-unknown-hermit/release/mutex", + "name": "Netbench UDP BW - Server", + "path": "target/x86_64-unknown-hermit/release/udp-server-bw", "group": "File Size", - "plot_group": "Benchmark File Size" + "plot_group": "Netbench Benchmark File Size" }, { - "name": "Micro Benchmarks", - "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks", - "iterations": 20, - "group": "Micro" + "name": "Netbench TCP Latency - Client", + "path": "target/x86_64-unknown-hermit/release/tcp-client-latency", + "group": "File Size", + "plot_group": "Netbench Benchmark File Size" }, { - "name": "Allocation Benchmarks", - "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 5G -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks", - "iterations": 30, - "group": "Allocations" + "name": "Netbench TCP Latency - Server", + "path": "target/x86_64-unknown-hermit/release/tcp-server-latency", + "group": "File Size", + "plot_group": "Netbench Benchmark File Size" }, { - "name": "Mutex Stress Test", - "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex", - "iterations": 40, - "group": "Mutex" + "name": "Netbench UDP Latency - Client", + "path": "target/x86_64-unknown-hermit/release/udp-client-latency", + "group": "File Size", + "plot_group": "Netbench Benchmark File Size" }, { - "name": "Pi Multithreaded Benchmark", - "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark", - "iterations": 15, - "group": "General", - "plot_group": "Multithreaded Pi Efficiency" + "name": "Netbench UDP Latency - Server", + "path": "target/x86_64-unknown-hermit/release/udp-server-latency", + "group": "File Size", + "plot_group": "Netbench Benchmark File Size" }, { "name": "Netbench TCP BW - Client", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 387552d30e..2c19e8645c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,6 @@ name: CI on: pull_request: merge_group: - env: GH_TOKEN: ${{ github.token }} @@ -252,56 +251,71 @@ jobs: if: matrix.arch == 'x86_64' benchmark: - name: Bench + name: Run Benchmarks runs-on: [self-hosted] + + strategy: + matrix: + include: + - benchmark-file: /kernel/.github/benchmarks/general.json + benchmark-build: true + - benchmark-file: /kernel/.github/benchmarks/misc.json + benchmark-build: false + - benchmark-file: /kernel/.github/benchmarks/netbench.json + benchmark-build: false + steps: - - name: Checkout hermit-rs - uses: actions/checkout@v4 - with: - repository: hermit-os/hermit-rs - submodules: true - - name: Remove hermit-kernel submodule - run: git rm -r kernel - working-directory: . - - name: Checkout hermit-kernel - uses: actions/checkout@v4 - with: - path: kernel - - name: Install QEMU - run: | - sudo apt-get update - sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm - - uses: mkroening/rust-toolchain-toml@main - - name: Install parallel - run: sudo apt install parallel -y - - name: Download loader - run: | - sudo apt install gh -y - gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64 - - name: Install uhyve - run: cargo +stable install --locked uhyve - - name: Check KVM availability - shell: bash - run: | - lscpu - kvm-ok - - name: Add wasm target - run: | - rustup component add llvm-tools - rustup target add wasm32-wasip1 - rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu - - name: Run benchmarks - uses: hermit-os/hermit-bench@kernel - with: - benchmark-file: /kernel/.github/benchmarks/bench.json - build-command: "HERMIT_LOG_LEVEL_FILTER=Off cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" - - name: Store benchmark result - # Only save the result if the commit is on main - # This is to avoid saving results for PRs - if: (github.event_name == 'push' && github.ref == 'refs/heads/main') - uses: hermit-os/github-action-benchmark@hermit-rs - with: - tool: 'hermit-bench' - output-file-path: results.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true + - name: Checkout hermit-rs + uses: actions/checkout@v4 + with: + repository: hermit-os/hermit-rs + submodules: true + - name: Remove hermit-kernel submodule + run: git rm -r kernel + - name: Checkout hermit-kernel + uses: actions/checkout@v4 + with: + path: kernel + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm parallel gh + - name: Download loader + run: gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64 + - uses: mkroening/rust-toolchain-toml@main + - name: Install uhyve + run: cargo +stable install --locked uhyve + - name: Check KVM availability + run: | + lscpu + kvm-ok + - name: Add wasm target + run: | + rustup component add llvm-tools + rustup target add wasm32-wasip1 + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - name: Run benchmarks + uses: hermit-os/hermit-bench@kernel + id: run-bench + with: + benchmark-file: ${{ matrix.benchmark-file }} + benchmark-build: ${{ matrix.benchmark-build }} + build-command: "HERMIT_LOG_LEVEL_FILTER=Off cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" + - name: Determine if benchmarks should be published + id: publish-benchmarks + run: | + if [[ "${{ github.event_name }}" == "push" && "${GITHUB_REF##*/}" == "main" ]]; then + echo "publish_benchmarks=true" >> $GITHUB_OUTPUT + else + echo "publish_benchmarks=false" >> $GITHUB_OUTPUT + fi + - name: Store benchmark results + uses: hermit-os/github-action-benchmark@hermit-rs + with: + tool: 'hermit-bench' + output-file-path: ${{ steps.run-bench.outputs.result-file }} + github-token: ${{ secrets.GITHUB_TOKEN }} + benchmark-data-dir-path: benchmarks + comment-always: true + auto-push: ${{ steps.publish-benchmarks.outputs.publish_benchmarks == 'true' }} + \ No newline at end of file From 158f85d3ae2038d4317907ba03ea5dd492b2739c Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Mon, 26 May 2025 19:21:02 +0200 Subject: [PATCH 04/17] feat(bench): Label benchmark sets --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c19e8645c..1479a2809e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,10 +258,13 @@ jobs: matrix: include: - benchmark-file: /kernel/.github/benchmarks/general.json + benchmark-matrix-name: General benchmark-build: true - benchmark-file: /kernel/.github/benchmarks/misc.json + benchmark-matrix-name: Misc benchmark-build: false - benchmark-file: /kernel/.github/benchmarks/netbench.json + benchmark-matrix-name: Netbench benchmark-build: false steps: @@ -317,5 +320,6 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} benchmark-data-dir-path: benchmarks comment-always: true + benchmark-matrix-name: ${{ matrix.benchmark-matrix-name }} auto-push: ${{ steps.publish-benchmarks.outputs.publish_benchmarks == 'true' }} \ No newline at end of file From a6d287701604a6cf0c85aa3e952d3461a71f2577 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Mon, 2 Jun 2025 10:53:38 +0200 Subject: [PATCH 05/17] feat(ci): move benchmark to seperate workflow --- .github/workflows/benchmark.yml | 86 +++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 74 ---------------------------- 2 files changed, 86 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..bab98c5e42 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,86 @@ +name: Benchmark + +on: + pull_request: + merge_group: + push: + branches: + - main + +env: + GH_TOKEN: ${{ github.token }} + +jobs: + benchmark: + name: Run Benchmarks + runs-on: [self-hosted] + + strategy: + matrix: + include: + - benchmark-file: /kernel/.github/benchmarks/general.json + benchmark-matrix-name: General + benchmark-build: true + - benchmark-file: /kernel/.github/benchmarks/misc.json + benchmark-matrix-name: Misc + benchmark-build: false + - benchmark-file: /kernel/.github/benchmarks/netbench.json + benchmark-matrix-name: Netbench + benchmark-build: false + + steps: + - name: Checkout hermit-rs + uses: actions/checkout@v4 + with: + repository: hermit-os/hermit-rs + submodules: true + - name: Remove hermit-kernel submodule + run: git rm -r kernel + - name: Checkout hermit-kernel + uses: actions/checkout@v4 + with: + path: kernel + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm parallel gh + - name: Download loader + run: gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64 + - uses: mkroening/rust-toolchain-toml@main + - name: Install uhyve + run: cargo +stable install --locked uhyve + - name: Check KVM availability + run: | + lscpu + kvm-ok + - name: Add wasm target + run: | + rustup component add llvm-tools + rustup target add wasm32-wasip1 + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - name: Run benchmarks + uses: hermit-os/hermit-bench@kernel + id: run-bench + with: + benchmark-file: ${{ matrix.benchmark-file }} + benchmark-build: ${{ matrix.benchmark-build }} + build-command: "HERMIT_LOG_LEVEL_FILTER=Off cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" + - name: Determine if benchmarks should be published + id: publish-benchmarks + run: | + if [[ "${{ github.event_name }}" == "push" && "${GITHUB_REF##*/}" == "main" ]]; then + echo "publish_benchmarks=true" >> $GITHUB_OUTPUT + else + echo "publish_benchmarks=false" >> $GITHUB_OUTPUT + fi + - name: Store benchmark results + uses: hermit-os/github-action-benchmark@hermit-rs + with: + tool: 'hermit-bench' + output-file-path: ${{ steps.run-bench.outputs.result-file }} + github-token: ${{ secrets.GITHUB_TOKEN }} + benchmark-data-dir-path: benchmarks + comment-always: true + benchmark-matrix-name: ${{ matrix.benchmark-matrix-name }} + auto-push: ${{ steps.publish-benchmarks.outputs.publish_benchmarks == 'true' }} + \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1479a2809e..e3ba24b2e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,77 +249,3 @@ jobs: if: matrix.arch == 'x86_64' - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} --package thread_test --smp 4 qemu ${{ matrix.flags }} if: matrix.arch == 'x86_64' - - benchmark: - name: Run Benchmarks - runs-on: [self-hosted] - - strategy: - matrix: - include: - - benchmark-file: /kernel/.github/benchmarks/general.json - benchmark-matrix-name: General - benchmark-build: true - - benchmark-file: /kernel/.github/benchmarks/misc.json - benchmark-matrix-name: Misc - benchmark-build: false - - benchmark-file: /kernel/.github/benchmarks/netbench.json - benchmark-matrix-name: Netbench - benchmark-build: false - - steps: - - name: Checkout hermit-rs - uses: actions/checkout@v4 - with: - repository: hermit-os/hermit-rs - submodules: true - - name: Remove hermit-kernel submodule - run: git rm -r kernel - - name: Checkout hermit-kernel - uses: actions/checkout@v4 - with: - path: kernel - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm parallel gh - - name: Download loader - run: gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64 - - uses: mkroening/rust-toolchain-toml@main - - name: Install uhyve - run: cargo +stable install --locked uhyve - - name: Check KVM availability - run: | - lscpu - kvm-ok - - name: Add wasm target - run: | - rustup component add llvm-tools - rustup target add wasm32-wasip1 - rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu - - name: Run benchmarks - uses: hermit-os/hermit-bench@kernel - id: run-bench - with: - benchmark-file: ${{ matrix.benchmark-file }} - benchmark-build: ${{ matrix.benchmark-build }} - build-command: "HERMIT_LOG_LEVEL_FILTER=Off cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" - - name: Determine if benchmarks should be published - id: publish-benchmarks - run: | - if [[ "${{ github.event_name }}" == "push" && "${GITHUB_REF##*/}" == "main" ]]; then - echo "publish_benchmarks=true" >> $GITHUB_OUTPUT - else - echo "publish_benchmarks=false" >> $GITHUB_OUTPUT - fi - - name: Store benchmark results - uses: hermit-os/github-action-benchmark@hermit-rs - with: - tool: 'hermit-bench' - output-file-path: ${{ steps.run-bench.outputs.result-file }} - github-token: ${{ secrets.GITHUB_TOKEN }} - benchmark-data-dir-path: benchmarks - comment-always: true - benchmark-matrix-name: ${{ matrix.benchmark-matrix-name }} - auto-push: ${{ steps.publish-benchmarks.outputs.publish_benchmarks == 'true' }} - \ No newline at end of file From 4916cd423fa256119f06906ada95d42a6a1b2a05 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Mon, 2 Jun 2025 12:14:45 +0200 Subject: [PATCH 06/17] fix(ci): add missing permissions for benchmark job --- .github/workflows/benchmark.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bab98c5e42..67a418732c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -14,6 +14,10 @@ jobs: benchmark: name: Run Benchmarks runs-on: [self-hosted] + permissions: + contents: write + pull-requests: write + repository-projects: write strategy: matrix: From 38d98ce8e251a816afc83a336968615d913228ee Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Mon, 2 Jun 2025 12:33:01 +0200 Subject: [PATCH 07/17] fix(ci): give PR writing permissions for forks --- .github/workflows/benchmark.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 67a418732c..48477f5eac 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,7 +1,7 @@ name: Benchmark on: - pull_request: + pull_request_target: merge_group: push: branches: @@ -15,9 +15,7 @@ jobs: name: Run Benchmarks runs-on: [self-hosted] permissions: - contents: write pull-requests: write - repository-projects: write strategy: matrix: From e5c2040c3744556b45ba4d946e6fa9d295a38c10 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Mon, 2 Jun 2025 12:59:46 +0200 Subject: [PATCH 08/17] fix(ci): prevent publish_docs from clearing benchmark data --- .github/workflows/publish_docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml index 2f930dc6c5..44e382b4a6 100644 --- a/.github/workflows/publish_docs.yml +++ b/.github/workflows/publish_docs.yml @@ -38,5 +38,6 @@ jobs: with: target_branch: gh-pages build_dir: target/x86_64-unknown-none/doc + keep_history: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 0994886ee87e9d45329070f03da7f70f51502f8b Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Tue, 3 Jun 2025 16:11:20 +0200 Subject: [PATCH 09/17] fix(bench): better qemu flags and benchmark arguments --- .github/benchmarks/general.json | 8 ++++---- .github/benchmarks/misc.json | 6 +++--- .github/benchmarks/netbench.json | 24 ++++++++++++------------ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/benchmarks/general.json b/.github/benchmarks/general.json index 8acc186828..d98ad30d69 100644 --- a/.github/benchmarks/general.json +++ b/.github/benchmarks/general.json @@ -1,7 +1,7 @@ [ { "name": "1 core", - "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", "external_time": true, "iterations": 25, "group": "General", @@ -9,7 +9,7 @@ }, { "name": "2 cores", - "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", "external_time": true, "iterations": 25, "group": "General", @@ -17,7 +17,7 @@ }, { "name": "4 cores", - "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", "external_time": true, "iterations": 25, "group": "General", @@ -25,7 +25,7 @@ }, { "name": "Pi Multithreaded Benchmark", - "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark", "iterations": 15, "group": "General", "plot_group": "Multithreaded Pi Efficiency" diff --git a/.github/benchmarks/misc.json b/.github/benchmarks/misc.json index cfa64bb7f1..03cd0a6989 100644 --- a/.github/benchmarks/misc.json +++ b/.github/benchmarks/misc.json @@ -19,19 +19,19 @@ }, { "name": "Micro Benchmarks", - "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks", + "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks", "iterations": 20, "group": "Micro" }, { "name": "Allocation Benchmarks", - "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 5G -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks", + "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 5G -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks", "iterations": 30, "group": "Allocations" }, { "name": "Mutex Stress Test", - "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex", + "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex", "iterations": 40, "group": "Mutex" } diff --git a/.github/benchmarks/netbench.json b/.github/benchmarks/netbench.json index 3214c59cee..5ce54a3d97 100644 --- a/.github/benchmarks/netbench.json +++ b/.github/benchmarks/netbench.json @@ -49,59 +49,59 @@ }, { "name": "Netbench TCP BW - Client", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 1000\"'", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 1000' 'sleep 2 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1048576 --rounds 1000\"'", "iterations": 3, "group": "Networking", "plot_group": "Netbench Bandwidth - 1000x1MB" }, { "name": "Netbench TCP BW - Server", - "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 1000\"'", + "command": "parallel ::: 'sleep 2 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1048576 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 1000\"'", "iterations": 3, "group": "Networking", "plot_group": "Netbench Bandwidth - 1000x1MB" }, { "name": "Netbench TCP Latency - Client", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 250' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 250\"'", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1048576 --rounds 250\"'", "iterations": 3, "group": "Networking", "plot_group": "Netbench Latency - 250x1MB" }, { "name": "Netbench TCP Latency - Server", - "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 250\"'", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 250\"'", "iterations": 3, "group": "Networking", "plot_group": "Netbench Latency - 250x1MB" }, { "name": "Netbench UDP BW - Server", - "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::7878-:7878,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000 \"'", - "iterations": 3, + "command": "parallel ::: 'sleep 2 && cargo run --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000\"'", + "iterations": 5, "group": "Networking", "plot_group": "Netbench Bandwidth - 1000x1472B" }, { "name": "Netbench UDP BW - Client", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000 \"'", - "iterations": 3, + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 2 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", + "iterations": 5, "group": "Networking", "plot_group": "Netbench Bandwidth - 1000x1472B" }, { "name": "Netbench UDP Latency - Client", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 10 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", - "iterations": 3, + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 2 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", + "iterations": 5, "group": "Networking", "plot_group": "Netbench Latency - 250x1472B" }, { "name": "Netbench UDP Latency - Server", - "command": "parallel ::: 'sleep 10 && cargo run --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu -- --nonblocking --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878,hostfwd=udp::7878-:7878,net=192.168.76.0/24,dhcpstart=192.168.76.9 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000 \"'", - "iterations": 3, + "command": "parallel ::: 'sleep 2 && cargo run --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000\"'", + "iterations": 5, "group": "Networking", "plot_group": "Netbench Latency - 250x1472B" } From 19df47f598a8164740e5fa4a1000f3d33ea77879 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Tue, 3 Jun 2025 16:11:59 +0200 Subject: [PATCH 10/17] cleanup: reformat benchmark workflow --- .github/workflows/benchmark.yml | 41 ++++++++++++++++----------------- .github/workflows/ci.yml | 2 +- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 48477f5eac..2df112bb98 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -7,6 +7,9 @@ on: branches: - main +permissions: + pull-requests: write + env: GH_TOKEN: ${{ github.token }} @@ -14,8 +17,6 @@ jobs: benchmark: name: Run Benchmarks runs-on: [self-hosted] - permissions: - pull-requests: write strategy: matrix: @@ -45,7 +46,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y qemu-system qemu-system-x86 qemu-kvm parallel gh + sudo apt-get install -y --no-install-recommends qemu-system-x86 parallel gh - name: Download loader run: gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64 - uses: mkroening/rust-toolchain-toml@main @@ -55,28 +56,27 @@ jobs: run: | lscpu kvm-ok - - name: Add wasm target - run: | - rustup component add llvm-tools - rustup target add wasm32-wasip1 - rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src + - uses: mkroening/rust-toolchain-toml@main + - run: rustup component add llvm-tools + working-directory: . + - run: rustup target add wasm32-wasip1 + working-directory: . + - uses: mkroening/rust-toolchain-toml@main + with: + toolchain-file: "kernel/rust-toolchain.toml" - name: Run benchmarks - uses: hermit-os/hermit-bench@kernel + uses: hermit-os/hermit-bench@main id: run-bench with: benchmark-file: ${{ matrix.benchmark-file }} benchmark-build: ${{ matrix.benchmark-build }} - build-command: "HERMIT_LOG_LEVEL_FILTER=Off cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" - - name: Determine if benchmarks should be published - id: publish-benchmarks - run: | - if [[ "${{ github.event_name }}" == "push" && "${GITHUB_REF##*/}" == "main" ]]; then - echo "publish_benchmarks=true" >> $GITHUB_OUTPUT - else - echo "publish_benchmarks=false" >> $GITHUB_OUTPUT - fi + build-command: "cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" - name: Store benchmark results - uses: hermit-os/github-action-benchmark@hermit-rs + uses: hermit-os/github-action-benchmark@main with: tool: 'hermit-bench' output-file-path: ${{ steps.run-bench.outputs.result-file }} @@ -84,5 +84,4 @@ jobs: benchmark-data-dir-path: benchmarks comment-always: true benchmark-matrix-name: ${{ matrix.benchmark-matrix-name }} - auto-push: ${{ steps.publish-benchmarks.outputs.publish_benchmarks == 'true' }} - \ No newline at end of file + auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3ba24b2e7..8ee3544a35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: pull_request: merge_group: - + env: GH_TOKEN: ${{ github.token }} RUSTFLAGS: -Dwarnings From 8929fd54b06a0cbe4c91f89835286d6ec4aa5ef4 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Tue, 3 Jun 2025 17:55:53 +0200 Subject: [PATCH 11/17] feat(bench): publish data to hermit-bench --- .github/workflows/benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2df112bb98..a3c758fee5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,7 +46,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends qemu-system-x86 parallel gh + sudo apt-get install -y --no-install-recommends qemu-system-x86 cpu-checker parallel gh - name: Download loader run: gh release download --repo hermit-os/loader --pattern hermit-loader-x86_64 - uses: mkroening/rust-toolchain-toml@main @@ -82,6 +82,7 @@ jobs: output-file-path: ${{ steps.run-bench.outputs.result-file }} github-token: ${{ secrets.GITHUB_TOKEN }} benchmark-data-dir-path: benchmarks + gh-repository: github.com/hermit-os/hermit-bench comment-always: true benchmark-matrix-name: ${{ matrix.benchmark-matrix-name }} auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} \ No newline at end of file From 611fd0946c207734e0e6211baf3b720779bb9d96 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Sun, 8 Jun 2025 11:52:35 +0200 Subject: [PATCH 12/17] feat(bench): prebuild netbench reference and tighten timing --- .github/benchmarks/netbench.json | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/benchmarks/netbench.json b/.github/benchmarks/netbench.json index 5ce54a3d97..aeb74b0edc 100644 --- a/.github/benchmarks/netbench.json +++ b/.github/benchmarks/netbench.json @@ -49,35 +49,40 @@ }, { "name": "Netbench TCP BW - Client", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 1000' 'sleep 2 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1048576 --rounds 1000\"'", - "iterations": 3, + "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 1000' 'sleep 1 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1048576 --rounds 1000\"'", + "iterations": 5, "group": "Networking", "plot_group": "Netbench Bandwidth - 1000x1MB" }, { "name": "Netbench TCP BW - Server", - "command": "parallel ::: 'sleep 2 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1048576 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 1000\"'", - "iterations": 3, + "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu", + "command": "parallel ::: 'sleep 1 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1048576 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 1000\"'", + "iterations": 5, "group": "Networking", "plot_group": "Netbench Bandwidth - 1000x1MB" }, { "name": "Netbench TCP Latency - Client", + "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu", "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1048576 --rounds 250\"'", - "iterations": 3, + "iterations": 5, "group": "Networking", "plot_group": "Netbench Latency - 250x1MB" }, { "name": "Netbench TCP Latency - Server", + "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu", "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 250\"'", - "iterations": 3, + "iterations": 5, "group": "Networking", "plot_group": "Netbench Latency - 250x1MB" }, { "name": "Netbench UDP BW - Server", - "command": "parallel ::: 'sleep 2 && cargo run --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000\"'", + "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu", + "command": "parallel ::: 'sleep 1 && cargo run --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000\"'", "iterations": 5, "group": "Networking", "plot_group": "Netbench Bandwidth - 1000x1472B" @@ -85,7 +90,8 @@ { "name": "Netbench UDP BW - Client", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 2 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", + "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 1 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", "iterations": 5, "group": "Networking", "plot_group": "Netbench Bandwidth - 1000x1472B" @@ -93,14 +99,16 @@ { "name": "Netbench UDP Latency - Client", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 2 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", + "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu", + "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 1 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", "iterations": 5, "group": "Networking", "plot_group": "Netbench Latency - 250x1472B" }, { "name": "Netbench UDP Latency - Server", - "command": "parallel ::: 'sleep 2 && cargo run --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000\"'", + "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu", + "command": "parallel ::: 'sleep 1 && cargo run --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000\"'", "iterations": 5, "group": "Networking", "plot_group": "Netbench Latency - 250x1472B" From 28c9a27dd424e98f990d6b61f45911235164a6ca Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Thu, 19 Jun 2025 10:09:14 +0200 Subject: [PATCH 13/17] bench: use enviorment specific token --- .github/workflows/benchmark.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a3c758fee5..2c445321a8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,6 +17,7 @@ jobs: benchmark: name: Run Benchmarks runs-on: [self-hosted] + environment: ${{ github.event_name == 'push' && 'benchmark' || 'default' }} strategy: matrix: @@ -80,9 +81,9 @@ jobs: with: tool: 'hermit-bench' output-file-path: ${{ steps.run-bench.outputs.result-file }} - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ github.event_name == 'push' && secrets.HERMIT_BENCH_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} benchmark-data-dir-path: benchmarks gh-repository: github.com/hermit-os/hermit-bench comment-always: true benchmark-matrix-name: ${{ matrix.benchmark-matrix-name }} - auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} \ No newline at end of file + auto-push: ${{ github.event_name == 'push' }} \ No newline at end of file From 226168ff6bda4214b6648ecc148e780f3397ac87 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Thu, 19 Jun 2025 11:18:37 +0200 Subject: [PATCH 14/17] fix(bench): update plot group labels --- .github/benchmarks/misc.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/benchmarks/misc.json b/.github/benchmarks/misc.json index 03cd0a6989..95740d31b4 100644 --- a/.github/benchmarks/misc.json +++ b/.github/benchmarks/misc.json @@ -3,19 +3,19 @@ "name": "alloc_benchmarks", "path": "target/x86_64-unknown-hermit/release/alloc_benchmarks", "group": "File Size", - "plot_group": "Benchmark File Size" + "plot_group": "Misc Benchmark File Size" }, { "name": "micro_benchmarks", "path": "target/x86_64-unknown-hermit/release/micro_benchmarks", "group": "File Size", - "plot_group": "Benchmark File Size" + "plot_group": "Misc Benchmark File Size" }, { "name": "mutex test", "path": "target/x86_64-unknown-hermit/release/mutex", "group": "File Size", - "plot_group": "Benchmark File Size" + "plot_group": "Misc Benchmark File Size" }, { "name": "Micro Benchmarks", From a76c663c440c9deb0e66c007fce0e69cf896cd0d Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Thu, 19 Jun 2025 11:20:58 +0200 Subject: [PATCH 15/17] bench: temporarily remove netbench --- .github/benchmarks/netbench.json | 116 ------------------------------- 1 file changed, 116 deletions(-) delete mode 100644 .github/benchmarks/netbench.json diff --git a/.github/benchmarks/netbench.json b/.github/benchmarks/netbench.json deleted file mode 100644 index aeb74b0edc..0000000000 --- a/.github/benchmarks/netbench.json +++ /dev/null @@ -1,116 +0,0 @@ -[ - { - "name": "Netbench TCP BW - Client", - "path": "target/x86_64-unknown-hermit/release/tcp-client-bw", - "group": "File Size", - "plot_group": "Netbench Benchmark File Size" - }, - { - "name": "Netbench TCP BW - Server", - "path": "target/x86_64-unknown-hermit/release/tcp-server-bw", - "group": "File Size", - "plot_group": "Netbench Benchmark File Size" - }, - { - "name": "Netbench UDP BW - Client", - "path": "target/x86_64-unknown-hermit/release/udp-client-bw", - "group": "File Size", - "plot_group": "Netbench Benchmark File Size" - }, - { - "name": "Netbench UDP BW - Server", - "path": "target/x86_64-unknown-hermit/release/udp-server-bw", - "group": "File Size", - "plot_group": "Netbench Benchmark File Size" - }, - { - "name": "Netbench TCP Latency - Client", - "path": "target/x86_64-unknown-hermit/release/tcp-client-latency", - "group": "File Size", - "plot_group": "Netbench Benchmark File Size" - }, - { - "name": "Netbench TCP Latency - Server", - "path": "target/x86_64-unknown-hermit/release/tcp-server-latency", - "group": "File Size", - "plot_group": "Netbench Benchmark File Size" - }, - { - "name": "Netbench UDP Latency - Client", - "path": "target/x86_64-unknown-hermit/release/udp-client-latency", - "group": "File Size", - "plot_group": "Netbench Benchmark File Size" - }, - { - "name": "Netbench UDP Latency - Server", - "path": "target/x86_64-unknown-hermit/release/udp-server-latency", - "group": "File Size", - "plot_group": "Netbench Benchmark File Size" - }, - { - "name": "Netbench TCP BW - Client", - "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 1000' 'sleep 1 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1048576 --rounds 1000\"'", - "iterations": 5, - "group": "Networking", - "plot_group": "Netbench Bandwidth - 1000x1MB" - }, - { - "name": "Netbench TCP BW - Server", - "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu", - "command": "parallel ::: 'sleep 1 && cargo run --manifest-path ./Cargo.toml --bin tcp-client-bw --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1048576 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 1000\"'", - "iterations": 5, - "group": "Networking", - "plot_group": "Netbench Bandwidth - 1000x1MB" - }, - { - "name": "Netbench TCP Latency - Client", - "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1048576 --rounds 250\"'", - "iterations": 5, - "group": "Networking", - "plot_group": "Netbench Latency - 250x1MB" - }, - { - "name": "Netbench TCP Latency - Server", - "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin tcp-client-latency --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1048576 --rounds 250' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/tcp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=tcp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1048576 --rounds 250\"'", - "iterations": 5, - "group": "Networking", - "plot_group": "Netbench Latency - 250x1MB" - }, - { - "name": "Netbench UDP BW - Server", - "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu", - "command": "parallel ::: 'sleep 1 && cargo run --manifest-path ./Cargo.toml --bin udp-client-bw --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000\"'", - "iterations": 5, - "group": "Networking", - "plot_group": "Netbench Bandwidth - 1000x1472B" - }, - { - - "name": "Netbench UDP BW - Client", - "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-bw --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 1 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-bw -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", - "iterations": 5, - "group": "Networking", - "plot_group": "Netbench Bandwidth - 1000x1472B" - }, - { - - "name": "Netbench UDP Latency - Client", - "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu", - "command": "parallel ::: 'cargo run --manifest-path ./Cargo.toml --bin udp-server-latency --release --target x86_64-unknown-linux-gnu -- --address 10.0.5.3 --bytes 1472 --rounds 1000' 'sleep 1 && sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-client-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::9975-:9975 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 127.0.0.1 --bytes 1472 --rounds 1000\"'", - "iterations": 5, - "group": "Networking", - "plot_group": "Netbench Latency - 250x1472B" - }, - { - "name": "Netbench UDP Latency - Server", - "pre_run_command": "cargo build --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu", - "command": "parallel ::: 'sleep 1 && cargo run --manifest-path ./Cargo.toml --bin udp-client-latency --release --target x86_64-unknown-linux-gnu -- --address 127.0.0.1 --bytes 1472 --rounds 1000' 'sudo qemu-system-x86_64 -display none -serial stdio -kernel hermit-loader-x86_64 -cpu host,migratable=no,+invtsc -enable-kvm -initrd target/x86_64-unknown-hermit/release/udp-server-latency -smp 2 -m 1024M -netdev user,id=u1,hostfwd=udp::7878-:7878 -device virtio-net-pci,netdev=u1,disable-legacy=on,packed=on,mq=on -append \"-- --address 10.0.5.3 --bytes 1472 --rounds 1000\"'", - "iterations": 5, - "group": "Networking", - "plot_group": "Netbench Latency - 250x1472B" - } -] From 84dbe037b5a35523188362c41729ebcaa5c9b675 Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Thu, 19 Jun 2025 11:56:15 +0200 Subject: [PATCH 16/17] bench: no default env --- .github/workflows/benchmark.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2c445321a8..f3975c12c3 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,7 +17,7 @@ jobs: benchmark: name: Run Benchmarks runs-on: [self-hosted] - environment: ${{ github.event_name == 'push' && 'benchmark' || 'default' }} + environment: ${{ github.event_name == 'push' && 'benchmark' || '' }} strategy: matrix: @@ -28,9 +28,6 @@ jobs: - benchmark-file: /kernel/.github/benchmarks/misc.json benchmark-matrix-name: Misc benchmark-build: false - - benchmark-file: /kernel/.github/benchmarks/netbench.json - benchmark-matrix-name: Netbench - benchmark-build: false steps: - name: Checkout hermit-rs From f5a019ff9ed094c9d6c9fc62255eda75d0ce696a Mon Sep 17 00:00:00 2001 From: CarlWachter Date: Wed, 25 Jun 2025 18:23:27 +0200 Subject: [PATCH 17/17] bench: build all benchmarks seperately --- .github/benchmarks/general.json | 24 ++++++++++-------------- .github/benchmarks/misc.json | 32 ++++++++++---------------------- .github/workflows/benchmark.yml | 9 +++++++-- 3 files changed, 27 insertions(+), 38 deletions(-) diff --git a/.github/benchmarks/general.json b/.github/benchmarks/general.json index d98ad30d69..5c94e7a35c 100644 --- a/.github/benchmarks/general.json +++ b/.github/benchmarks/general.json @@ -1,6 +1,8 @@ [ { "name": "1 core", + "bin": "startup_benchmark", + "manifest_path": "Cargo.toml", "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", "external_time": true, "iterations": 25, @@ -9,6 +11,8 @@ }, { "name": "2 cores", + "bin": "startup_benchmark", + "manifest_path": "Cargo.toml", "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", "external_time": true, "iterations": 25, @@ -17,6 +21,8 @@ }, { "name": "4 cores", + "bin": "startup_benchmark", + "manifest_path": "Cargo.toml", "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 128M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/startup_benchmark", "external_time": true, "iterations": 25, @@ -25,21 +31,11 @@ }, { "name": "Pi Multithreaded Benchmark", + "bin": "multithreaded_benchmark", + "manifest_path": "Cargo.toml", "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/multithreaded_benchmark", - "iterations": 15, + "iterations": 25, "group": "General", "plot_group": "Multithreaded Pi Efficiency" - }, - { - "name": "Pi Multithreaded Benchmark", - "path": "target/x86_64-unknown-hermit/release/multithreaded_benchmark", - "group": "File Size", - "plot_group": "General Benchmark File Size" - }, - { - "name": "Startup Benchmark", - "path": "target/x86_64-unknown-hermit/release/startup_benchmark", - "group": "File Size", - "plot_group": "General Benchmark File Size" } -] +] \ No newline at end of file diff --git a/.github/benchmarks/misc.json b/.github/benchmarks/misc.json index 95740d31b4..50d70cff7f 100644 --- a/.github/benchmarks/misc.json +++ b/.github/benchmarks/misc.json @@ -1,38 +1,26 @@ [ - { - "name": "alloc_benchmarks", - "path": "target/x86_64-unknown-hermit/release/alloc_benchmarks", - "group": "File Size", - "plot_group": "Misc Benchmark File Size" - }, - { - "name": "micro_benchmarks", - "path": "target/x86_64-unknown-hermit/release/micro_benchmarks", - "group": "File Size", - "plot_group": "Misc Benchmark File Size" - }, - { - "name": "mutex test", - "path": "target/x86_64-unknown-hermit/release/mutex", - "group": "File Size", - "plot_group": "Misc Benchmark File Size" - }, { "name": "Micro Benchmarks", + "bin": "micro_benchmarks", + "manifest_path": "benches/micro/Cargo.toml", "command": "sudo qemu-system-x86_64 -display none -smp 4 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/micro_benchmarks", - "iterations": 20, + "iterations": 25, "group": "Micro" }, { "name": "Allocation Benchmarks", + "bin": "alloc_benchmarks", + "manifest_path": "benches/alloc/Cargo.toml", "command": "sudo qemu-system-x86_64 -display none -smp 1 -m 5G -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/alloc_benchmarks", - "iterations": 30, + "iterations": 50, "group": "Allocations" }, { "name": "Mutex Stress Test", + "bin": "mutex", + "manifest_path": "benches/mutex/Cargo.toml", "command": "sudo qemu-system-x86_64 -display none -smp 2 -m 512M -serial stdio -enable-kvm -cpu host,migratable=no,+invtsc -kernel hermit-loader-x86_64 -initrd target/x86_64-unknown-hermit/release/mutex", - "iterations": 40, + "iterations": 50, "group": "Mutex" } -] +] \ No newline at end of file diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f3975c12c3..4b5c6e9f07 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -66,13 +66,18 @@ jobs: - uses: mkroening/rust-toolchain-toml@main with: toolchain-file: "kernel/rust-toolchain.toml" + - name: Set up tap device + if: matrix.benchmark-matrix-name == 'Netbench' + run: | + sudo ip tuntap add tap10 mode tap + sudo ip addr add 10.0.5.1/24 broadcast 10.0.5.255 dev tap10 + sudo ip link set dev tap10 + echo 1 | sudo tee /proc/sys/net/ipv4/conf/tap10/proxy_arp - name: Run benchmarks uses: hermit-os/hermit-bench@main id: run-bench with: benchmark-file: ${{ matrix.benchmark-file }} - benchmark-build: ${{ matrix.benchmark-build }} - build-command: "cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --release" - name: Store benchmark results uses: hermit-os/github-action-benchmark@main with: