Fix firmware inclusion in initrd

build.zig

@@ -1,4 +1,7 @@
 const std = @import("std");
+const utils = @import("./src/utils.zig");
+
+const TBOOT_INITRD_NAME = "tboot-initrd";
 
 pub fn build(b: *std.Build) !void {
     const target = b.standardTargetOptions(
@@ -46,84 +49,19 @@ pub fn build(b: *std.Build) !void {
     });
     const linux_headers_module = linux_headers_translated.addModule("linux_headers");
 
-    if (with_loader) {
-        const tboot_loader = b.addExecutable(.{
-            .name = "tboot-loader",
-            .root_source_file = b.path("src/tboot-loader.zig"),
-            .target = target,
-            .optimize = tboot_loader_optimize,
-            .strip = optimize != std.builtin.OptimizeMode.Debug,
-        });
-        tboot_loader.root_module.addAnonymousImport("test_key", .{
-            .root_source_file = b.path("tests/keys/tboot/key.der"),
-        });
-        tboot_loader.root_module.addImport("linux_headers", linux_headers_module);
-
-        var run_tboot_initrd = b.addSystemCommand(&.{"tboot-initrd"});
-
-        // TODO(jared): Would be nicer to have generic
-        // --file=tboot_loader:/init CLI interface, but don't know how to
-        // obtain path and string format it into that form. Further, would
-        // be nicer to not shell-out to a separate tool at all and just do
-        // the CPIO generation in here.
-        run_tboot_initrd.addPrefixedFileArg("-i", tboot_loader.getEmittedBin());
-
-        if (firmware_directory) |directory| {
-            const directory_ = b.addWriteFiles().addCopyDirectory(
-                .{ .cwd_relative = directory },
-                "",
-                .{},
-            );
-
-            run_tboot_initrd.addPrefixedDirectoryArg("-d", directory_);
-        }
-
-        const cpio_output_file = run_tboot_initrd.addPrefixedOutputFileArg(
-            "-o",
-            "tboot-loader.cpio",
-        );
-
-        run_tboot_initrd.expectExitCode(0);
-
-        const cpio_archive = b.addInstallFile(
-            cpio_output_file,
-            "tboot-loader.cpio",
-        );
-
-        // install the cpio archive during "zig build install"
-        b.getInstallStep().dependOn(&cpio_archive.step);
-
-        const runner_tool = b.addRunArtifact(b.addExecutable(.{
-            .name = "tboot-runner",
-            .target = b.graph.host,
-            .root_source_file = b.path("src/runner.zig"),
-        }));
-        runner_tool.step.dependOn(&cpio_archive.step);
-        runner_tool.addArg(b.makeTempPath());
-        runner_tool.addFileArg(cpio_archive.source);
-
-        var env = try std.process.getEnvMap(b.allocator);
-        defer env.deinit();
-
-        if (env.get("TINYBOOT_KERNEL")) |kernel| {
-            runner_tool.addArg(kernel);
-        }
-
-        // extra args passed through to qemu
-        if (b.args) |args| {
-            runner_tool.addArgs(args);
-        }
-
-        const run_step = b.step("run", "Run in qemu");
-        run_step.dependOn(&runner_tool.step);
-    }
+    // For re-usage with building the tboot-loader initrd, if we are also
+    // building tools.
+    var maybe_tboot_initrd_tool: ?*std.Build.Step.Compile = null;
 
     if (with_tools) {
-        const tboot_initrd_tool = b.addExecutable(.{
-            .name = "tboot-initrd",
+        maybe_tboot_initrd_tool = b.addExecutable(.{
+            .name = TBOOT_INITRD_NAME,
             .target = target,
+            .optimize = optimize,
             .root_source_file = b.path("src/tboot-initrd.zig"),
         });
+
+        var tboot_initrd_tool = maybe_tboot_initrd_tool.?;
         tboot_initrd_tool.linkLibC();
         tboot_initrd_tool.linkSystemLibrary("liblzma");
         tboot_initrd_tool.root_module.addImport("clap", clap.module("clap"));
@@ -146,6 +84,7 @@ pub fn build(b: *std.Build) !void {
             .optimize = optimize,
             .strip = optimize != std.builtin.OptimizeMode.Debug,
         });
+        tboot_bless_boot_generator.root_module.addImport("clap", clap.module("clap"));
         b.installArtifact(tboot_bless_boot_generator);
 
         const tboot_sign = b.addExecutable(.{
@@ -195,6 +134,83 @@ pub fn build(b: *std.Build) !void {
         b.installArtifact(tboot_vpd);
     }
 
+    if (with_loader) {
+        const tboot_loader = b.addExecutable(.{
+            .name = "tboot-loader",
+            .root_source_file = b.path("src/tboot-loader.zig"),
+            .target = target,
+            .optimize = tboot_loader_optimize,
+            .strip = optimize != std.builtin.OptimizeMode.Debug,
+        });
+        tboot_loader.root_module.addAnonymousImport("test_key", .{
+            .root_source_file = b.path("tests/keys/tboot/key.der"),
+        });
+        tboot_loader.root_module.addImport("linux_headers", linux_headers_module);
+
+        // First look for a local build of tboot-initrd, helpful for iteration
+        // on the tool itself, otherwise use the one that exists on $PATH.
+        var run_tboot_initrd = if (maybe_tboot_initrd_tool) |tboot_initrd_tool|
+            b.addRunArtifact(tboot_initrd_tool)
+        else
+            b.addSystemCommand(&.{TBOOT_INITRD_NAME});
+
+        // TODO(jared): Would be nicer to have generic
+        // --file=tboot_loader:/init CLI interface, but don't know how to
+        // obtain path and string format it into that form. Further, would
+        // be nicer to not shell-out to a separate tool at all and just do
+        // the CPIO generation in here.
+        run_tboot_initrd.addPrefixedFileArg("-i", tboot_loader.getEmittedBin());
+
+        if (firmware_directory) |directory| {
+            const directory_ = b.addWriteFiles().addCopyDirectory(
+                .{ .cwd_relative = directory },
+                "",
+                .{},
+            );
+
+            run_tboot_initrd.addPrefixedDirectoryArg("-d", directory_);
+        }
+
+        const cpio_output_file = run_tboot_initrd.addPrefixedOutputFileArg(
+            "-o",
+            "tboot-loader.cpio",
+        );
+
+        run_tboot_initrd.expectExitCode(0);
+
+        const cpio_archive = b.addInstallFile(
+            cpio_output_file,
+            "tboot-loader.cpio",
+        );
+
+        // install the cpio archive during "zig build install"
+        b.getInstallStep().dependOn(&cpio_archive.step);
+
+        const runner_tool = b.addRunArtifact(b.addExecutable(.{
+            .name = "tboot-runner",
+            .target = b.graph.host,
+            .root_source_file = b.path("src/runner.zig"),
+        }));
+        runner_tool.step.dependOn(&cpio_archive.step);
+        runner_tool.addArg(b.makeTempPath());
+        runner_tool.addFileArg(cpio_archive.source);
+
+        var env = try std.process.getEnvMap(b.allocator);
+        defer env.deinit();
+
+        if (env.get("TINYBOOT_KERNEL")) |kernel| {
+            runner_tool.addArg(kernel);
+        }
+
+        // extra args passed through to qemu
+        if (b.args) |args| {
+            runner_tool.addArgs(args);
+        }
+
+        const run_step = b.step("run", "Run in qemu");
+        run_step.dependOn(&runner_tool.step);
+    }
+
     const unit_tests = b.addTest(.{
         .root_source_file = b.path("src/test.zig"),
         .target = target,

compress-firmware.nix

@@ -0,0 +1,51 @@
+# This is adapted from https://github.com/nixos/nixpkgs/blob/5e4947a31bd21b33ccabcb9ff06d685b68d1e9c4/pkgs/build-support/kernel/compress-firmware.nix,
+# but dereferences all symlinks so that the zig build system is capable of
+# including all paths we want. See https://github.com/ziglang/zig/blob/53216d2f22053ca94a68f5da234038c01f73d60f/lib/std/Build/Step/WriteFile.zig#L232.
+
+{
+  runCommand,
+  lib,
+  zstd,
+}:
+
+firmwares:
+
+let
+  compressor = {
+    ext = "xz";
+    nativeBuildInputs = [ ];
+    cmd = file: target: ''xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${file}" > "${target}"'';
+  };
+in
+
+runCommand "firmware-xz"
+  {
+    allowedRequisites = [ ];
+    inherit (compressor) nativeBuildInputs;
+  }
+  (
+    lib.concatLines (
+      map (firmware: ''
+        mkdir -p $out/lib
+        (cd ${firmware} && find lib/firmware -type d -print0) |
+            (cd $out && xargs -0 mkdir -v --)
+        (cd ${firmware} && find lib/firmware -type f -print0) |
+            (cd $out && xargs -0rtP "$NIX_BUILD_CORES" -n1 \
+                sh -c '${compressor.cmd "${firmware}/$1" "$1.${compressor.ext}"}' --)
+        (cd ${firmware} && find lib/firmware -type l) | while read link; do
+            target="$(readlink "${firmware}/$link")"
+            if [ -f "${firmware}/$link" ]; then
+              cp -vL -- "''${target/^${firmware}/$out}.${compressor.ext}" "$out/$link.${compressor.ext}"
+            else
+              echo HI
+              cp -vrL -- "''${target/^${firmware}/$out}" "$out/$link"
+            fi
+        done
+
+        find $out
+
+        echo "Checking for broken symlinks:"
+        find -L $out -type l -print -execdir false -- '{}' '+'
+      '') firmwares
+    )
+  )

options.nix

@@ -193,14 +193,7 @@ in
       firmware = mkOption {
         type = types.listOf types.package;
         default = [ ];
-        apply =
-          list:
-          pkgs.buildEnv {
-            name = "firmware";
-            paths = map pkgs.compressFirmwareXz list;
-            pathsToLink = [ "/lib/firmware" ];
-            ignoreCollisions = true;
-          };
+        apply = pkgs.callPackage ./compress-firmware.nix { };
       };
     };
 

pkgs/linux/default.nix

@@ -25,7 +25,25 @@ stdenv.mkDerivation {
   postConfigure = ''
     cat $kconfigPath $extraConfigPath > all.config
     make -j$NIX_BUILD_CORES ARCH=${stdenv.hostPlatform.linuxArch} KCONFIG_ALLCONFIG=1 allnoconfig
-    bash ${./check_config.bash} all.config .config
+
+    start_config=all.config
+    end_config=.config
+
+    missing=()
+    while read -r line; do
+      if ! grep --silent "$line" "$end_config"; then
+        missing+=("$line")
+      fi
+    done <"$start_config"
+
+    if [[ ''${#missing[@]} -gt 0 ]]; then
+      echo
+      for line in "''${missing[@]}"; do
+        echo "\"$line\" not found in final config!"
+      done
+      echo
+      exit 1
+    fi
   '';
   buildFlags = [
     "DTC_FLAGS=-@"