Restore squashfs support

README.md

@@ -1,5 +1,5 @@
 Isolate NixOS QEMU VMs from each other and from the host by using a
-squashfs for the VM's /nix/store that contains only the VM's dependencies
+private /nix/store image that contains only the VM's dependencies
 (like the installer has) rather than a virtio mount of the host's entire
 /nix/store.
 

checks/mount-grep.nix

@@ -8,6 +8,14 @@ pkgs: {
   nodes = {
     shared = _: { };
     private = _: { imports = [ ../modules/qemu-vm-isolation.nix ]; };
+    privateErofs = _: {
+      imports = [ ../modules/qemu-vm-isolation.nix ];
+      virtualisation.qemu.isolation.nixStoreFilesystemType = "erofs";
+    };
+    privateSquash = _: {
+      imports = [ ../modules/qemu-vm-isolation.nix ];
+      virtualisation.qemu.isolation.nixStoreFilesystemType = "squashfs";
+    };
     useNixStoreImage = {
       virtualisation = {
         sharedDirectories = pkgs.lib.mkForce { };
@@ -18,16 +26,14 @@ pkgs: {
 
   testScript = ''
     start_all()
-    shared.wait_for_unit("multi-user.target")
-    private.wait_for_unit("multi-user.target")
-    useNixStoreImage.wait_for_unit("multi-user.target")
+    for machine in [shared, private, privateErofs, privateSquash, useNixStoreImage]:
+      machine.wait_for_unit("multi-user.target")
 
     shared.succeed("[[ $(mount | grep -c virt) -gt 0 ]]")
-    private.succeed("[[ $(mount | grep -c virt) -eq 0 ]]")
-    useNixStoreImage.succeed("[[ $(mount | grep -c virt) -eq 0 ]]")
-
     shared.succeed("[[ -e ${pkgs.pv} ]]")
-    private.fail("[[ -e ${pkgs.pv} ]]")
-    useNixStoreImage.fail("[[ -e ${pkgs.pv} ]]")
+
+    for machine in [private, privateErofs, privateSquash, useNixStoreImage]:
+      machine.succeed("[[ $(mount | grep -c virt) -eq 0 ]]")
+      machine.fail("[[ -e ${pkgs.pv} ]]")
   '';
 }

modules/libblkid-squashfs-nix-store-kludge.patch

@@ -0,0 +1,19 @@
+# This dubious kludge results from
+# https://github.com/NixOS/nixpkgs/pull/236656 requiring filesystems to have labels and
+# https://github.com/plougher/squashfs-tools/issues/59 squashfs not supporting labels.
+diff --git a/libblkid/src/superblocks/squashfs.c b/libblkid/src/superblocks/squashfs.c
+index 4db842493..ed7465882 100644
+--- a/libblkid/src/superblocks/squashfs.c
++++ b/libblkid/src/superblocks/squashfs.c
+@@ -45,6 +45,11 @@ static int probe_squashfs(blkid_probe pr, const struct blkid_idmag *mag)
+
+ 	blkid_probe_sprintf_version(pr, "%u.%u", vermaj, vermin);
+
++	{
++		char label_kludge[] = "nix-store";
++		blkid_probe_set_label(pr, label_kludge, sizeof(label_kludge));
++	}
++
+ 	return 0;
+ }
+

modules/qemu-vm-isolation.nix

@@ -1,56 +1,121 @@
 { config, lib, modulesPath, pkgs, ... }:
 let
-  inherit (lib) findSingle mkForce mkIf mkMerge mkVMOverride;
+  inherit (lib)
+    escapeShellArg mkForce mkIf mkMerge mkOption mkVMOverride optional;
 
-  lookupDriveDeviceName = driveName: driveList:
-    (findSingle (drive: drive.name == driveName)
-      (throw "Drive ${driveName} not found")
-      (throw "Multiple drives named ${driveName}") driveList).device;
+  cfg = config.virtualisation.qemu.isolation;
 
   storeMountPath = if config.virtualisation.writableStore then
     "/nix/.ro-store"
   else
     "/nix/store";
 
-in mkMerge [
-  {
+  hostPkgs = config.virtualisation.host.pkgs;
 
-    boot.initrd.availableKernelModules = [ "squashfs" ];
+  storeContents =
+    hostPkgs.closureInfo { rootPaths = config.virtualisation.additionalPaths; };
 
-    fileSystems = mkVMOverride {
-      "${storeMountPath}" = {
-        device =
-          lookupDriveDeviceName "nixstore" config.virtualisation.qemu.drives;
-        fsType = "squashfs";
-        options = [ "ro" ];
-        neededForBoot = true;
-      };
+  nixStoreImages = {
+    ext4 = "${
+        import (modulesPath + "/../lib/make-disk-image.nix") {
+          inherit pkgs config lib;
+          additionalPaths = [ storeContents ];
+          onlyNixStore = true;
+          label = "nix-store";
+          partitionTableType = "none";
+          installBootLoader = false;
+          diskSize = "auto";
+          additionalSpace = "0M";
+          copyChannel = false;
+        }
+      }/nixos.img";
+    erofs = "${
+        hostPkgs.runCommand "nix-store-image" { } ''
+          mkdir $out
+          cd ${builtins.storeDir}
+          ${hostPkgs.erofs-utils}/bin/mkfs.erofs \
+            --force-uid=0 \
+            --force-gid=0 \
+            -L nix-store \
+            -U eb176051-bd15-49b7-9e6b-462e0b467019 \
+            -T 0 \
+            --exclude-regex="$(
+              <${storeContents}/store-paths \
+                sed -e 's^.*/^^g' \
+              | cut -c -10 \
+              | ${hostPkgs.python3}/bin/python -c ${
+                escapeShellArg (builtins.readFile
+                  (modulesPath + "/virtualisation/includes-to-excludes.py"))
+              } )" \
+            $out/nix-store.img \
+            .
+        ''
+      }/nix-store.img";
+    squashfs =
+      "${hostPkgs.callPackage (modulesPath + "/../lib/make-squashfs.nix") {
+        storeContents = config.virtualisation.additionalPaths;
+      }}";
+  };
+
+in {
+  options = {
+    virtualisation.qemu.isolation.nixStoreFilesystemType = mkOption {
+      description = ''
+        What filesystem to use for the guest's Nix store.
+
+        erofs is more compact than ext4, but less mature.
+
+        squashfs support currently requires a dubious kludge that results in these
+        VMs not being able to mount any other squashfs volumes besides the nix store.
+      '';
+      type = lib.types.enum [ "ext4" "erofs" "squashfs" ];
+      default = "ext4";
     };
+  };
+  config = mkMerge [
+    {
+      boot.initrd.kernelModules =
+        optional (cfg.nixStoreFilesystemType == "erofs") "erofs";
 
-    system.build.squashfsStore =
-      pkgs.callPackage (modulesPath + "/../lib/make-squashfs.nix") {
-        storeContents = config.virtualisation.additionalPaths;
+      nixpkgs.overlays = optional (cfg.nixStoreFilesystemType == "squashfs")
+        (final: prev: {
+          util-linux = prev.util-linux.overrideAttrs (old: {
+            patches = (old.patches or [ ])
+              ++ [ ./libblkid-squashfs-nix-store-kludge.patch ];
+          });
+        });
+
+      fileSystems = mkVMOverride {
+        "${storeMountPath}" = {
+          fsType = cfg.nixStoreFilesystemType;
+          options = [ "ro" ];
+          neededForBoot = true;
+          label = "nix-store";
+        };
       };
 
-    virtualisation = {
+      system.build.nixStoreImage =
+        nixStoreImages."${cfg.nixStoreFilesystemType}";
 
-      sharedDirectories = mkForce { };
+      virtualisation = {
 
-      qemu.drives = [{
-        name = "nixstore";
-        file = "${config.system.build.squashfsStore}";
-        driveExtraOpts = {
-          format = "raw";
-          read-only = "on";
-          werror = "report";
-        };
-      }];
+        sharedDirectories = mkForce { };
 
-    };
-  }
-  (mkIf (lib.version < "23.05") {
-    # This should always have been the default.
-    virtualisation.bootDevice =
-      lookupDriveDeviceName "root" config.virtualisation.qemu.drives;
-  })
-]
+        qemu.drives = [{
+          file = config.system.build.nixStoreImage;
+          driveExtraOpts = {
+            format = "raw";
+            read-only = "on";
+            werror = "report";
+          };
+        }];
+
+      };
+    }
+    (mkIf (cfg.nixStoreFilesystemType == "ext4") {
+      # We use this to disable fsck runs on the ext4 nix store image because stage-1
+      # fsck crashes (maybe because the device is read-only?), halting boot.
+      boot.initrd.checkJournalingFS = false;
+    })
+  ];
+}