feat: use rules_qemu for local system tests

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "image": "ghcr.io/dfinity/ic-dev@sha256:b0c0702f842bcdf82ab48f82e70d204398e8b6cc0b0fb29b92552a7704862d54",
+  "image": "ghcr.io/dfinity/ic-dev@sha256:877c1e0c4426d9542f6ff88b419be78fd9d7d235d51c2996246d012b9e4a43dc",
   "remoteUser": "ubuntu",
   "privileged": true,
   "runArgs": [

.github/workflows/api-bn-recovery-test.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on:
       labels: dind-large
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host
         --mount type=tmpfs,target="/home/buildifier/.local/share/containers"

.github/workflows/ci-main.yml

@@ -33,7 +33,7 @@ jobs:
     runs-on: &dind-large-setup
       labels: dind-large
     container: &container-setup
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
          -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 90

.github/workflows/ci-pr-only.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: &dind-small-setup
       labels: dind-small
     container: &container-setup
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
          -e NODE_NAME --mount type=tmpfs,target="/tmp/containers"
     steps:

.github/workflows/container-api-bn-recovery.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on:
       labels: dind-large
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host
         --mount type=tmpfs,target="/home/buildifier/.local/share/containers"

.github/workflows/container-scan-nightly.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on:
       labels: dind-large
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 60

.github/workflows/local-system-tests.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on:
       labels: dind-large
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
          -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 600

.github/workflows/pocket-ic-tests-windows.yml

@@ -45,7 +45,7 @@ jobs:
   bazel-build-pocket-ic:
     name: Bazel Build PocketIC
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 90

.github/workflows/rate-limits-backend-release.yml

@@ -32,7 +32,7 @@ jobs:
       labels: dind-large
 
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers"
 

.github/workflows/release-testing.yml

@@ -35,7 +35,7 @@ jobs:
       group: dm1
       labels: dind-large
     container: &container-setup
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
          -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 180

.github/workflows/rosetta-release.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on:
       labels: dind-large
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     environment: DockerHub

.github/workflows/salt-sharing-canister-release.yml

@@ -32,7 +32,7 @@ jobs:
       labels: dind-large
 
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers"
 

.github/workflows/schedule-daily.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: &dind-large-setup
       labels: dind-large
     container: &container-setup
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 720 # 12 hours

.github/workflows/schedule-rust-bench.yml

@@ -24,7 +24,7 @@ jobs:
       # see linux-x86-64 runner group
       labels: rust-benchmarks
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       # running on bare metal machine using ubuntu user
       options: --user ubuntu --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 720 # 12 hours

.github/workflows/system-tests-benchmarks-nightly.yml

@@ -17,7 +17,7 @@ jobs:
       group: dm1
       labels: dind-large
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers"
     timeout-minutes: 480

.github/workflows/update-mainnet-canister-revisions.yaml

@@ -25,7 +25,7 @@ jobs:
       labels: dind-small
     environment: CREATE_PR
     container:
-      image: ghcr.io/dfinity/ic-build@sha256:6766ee9ed7fff7de14cf8ed86c6db8adc0d6b90788234440bd18c07743099ac0
+      image: ghcr.io/dfinity/ic-build@sha256:d760427d72cf6420c84f7558af39e9eb3d8a0d26a15892fa1500c4f592a478ae
       options: >-
         -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers"
     env:

MODULE.bazel

@@ -56,6 +56,18 @@ archive_override(
 bazel_dep(name = "pigz", version = "2.8.bcr.1")  # (parallel) gzip
 bazel_dep(name = "zstd", version = "1.5.7.bcr.1")  # zstd can change format across versions
 
+# QEMU, for the local system test backend. We only use the prebuilt (hermetic)
+# qemu binaries; calling qemu is handled by the system test driver.
+bazel_dep(name = "rules_qemu", version = "0.3.0")
+
+qemu = use_extension("@rules_qemu//qemu/extension:qemu.bzl", "qemu")
+use_repo(
+    qemu,
+    "qemu_img_prebuilt_linux_amd64",
+    "qemu_system_bin_prebuilt_linux_amd64_x86_64_softmmu",
+    "qemu_system_data_prebuilt_linux_amd64",
+)
+
 # Rust
 
 include("//bazel:rust.MODULE.bazel")

ci/container/TAG

@@ -1 +1 @@
-a76da6c4913ea85f179201fa22ca3552e4ad455ebe9a95c1f7c88992d0b0c4cd
+abbba8fa906b3d98553d787112d7ce0041af806f8758e28bbbe0ea5084a5abe3

ci/container/files/packages.common

@@ -48,8 +48,6 @@ iputils-ping # for ping6 required by the bazel-test-bare-metal
 # Local system-test backend (see rs/tests/driver/src/driver/local_backend.rs),
 # which launches KVM-accelerated QEMU VMs. `ovmf` provides the guests' UEFI
 # firmware.
-qemu-system-x86
-qemu-utils
 ovmf
 # Provides `capsh` (source of the baked-in /usr/local/bin/ic-net-admin launcher)
 # plus `setcap`/`getcap`; see ci/container/Dockerfile.

rs/tests/driver/src/driver/local_backend.rs

@@ -958,7 +958,7 @@ impl LocalBackend {
                 primary_disk.display(),
                 base.display()
             );
-            let mut cmd = Command::new("qemu-img");
+            let mut cmd = Command::new(get_dependency_path_from_env("ENV_DEPS__QEMU_IMG_PATH"));
             cmd.arg("create")
                 .arg("-q")
                 .arg("-f")
@@ -1081,6 +1081,12 @@ impl LocalBackend {
             }};
         }
 
+        // Specify the path for qemu data (e.g. the virtio-net PXE ROM), otherwise qemu tries to
+        // find it at /usr/share/qemu.
+        arg!(
+            "-L",
+            get_dependency_path_from_env("ENV_DEPS__QEMU_SYSTEM_DATA_PATH").display()
+        );
         arg!("-name", format!("guest={domain_name}"));
         arg!("-machine", "q35,accel=kvm");
         arg!("-cpu", "host");
@@ -1193,13 +1199,15 @@ impl LocalBackend {
         // With `-daemonize` the foreground process exits once the guest is up (or
         // nonzero, having printed the error to stderr, if startup failed), while
         // the VM runs on as a reparented process.
-        let output = Command::new("qemu-system-x86_64")
-            .args(&args)
-            .stdin(Stdio::null())
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped())
-            .output()
-            .with_context(|| format!("launching qemu-system-x86_64 for {domain_name}"))?;
+        let output = Command::new(get_dependency_path_from_env(
+            "ENV_DEPS__QEMU_SYSTEM_X86_64_PATH",
+        ))
+        .args(&args)
+        .stdin(Stdio::null())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .output()
+        .with_context(|| format!("launching qemu-system-x86_64 for {domain_name}"))?;
         if !output.status.success() {
             bail!(
                 "qemu-system-x86_64 failed to start VM {domain_name} (status {}): {}",

rs/tests/system_tests.bzl

@@ -293,6 +293,9 @@ def system_test(
     _local_only_deps["ENV_DEPS__UNIVERSAL_VM_DISK_IMG_PATH"] = "@farm_universal_vm_img//file"
     _local_only_deps["ENV_DEPS__PROMETHEUS_VM_DISK_IMG_PATH"] = "@farm_prometheus_vm_img//file"
     _local_only_deps["ENV_DEPS__DNSMASQ_PATH"] = "@dnsmasq//:dnsmasq"
+    _local_only_deps["ENV_DEPS__QEMU_IMG_PATH"] = "@qemu_img_prebuilt_linux_amd64//:qemu-img"
+    _local_only_deps["ENV_DEPS__QEMU_SYSTEM_X86_64_PATH"] = "@qemu_system_bin_prebuilt_linux_amd64_x86_64_softmmu//:qemu-system-x86_64"
+    _local_only_deps["ENV_DEPS__QEMU_SYSTEM_DATA_PATH"] = "@qemu_system_data_prebuilt_linux_amd64//:qemu-system-data"
 
     local_dep_env = {
         name: "$(rootpath {})".format(dep)