Make CPIO tool capable of adding files from arbitrary directories

This will allow for us to not have to concatenate multiple CPIOs
together just to add some firmware from linux-firmware.

build.zig

@@ -13,6 +13,7 @@ pub fn build(b: *std.Build) !void {
 
     const with_loader = b.option(bool, "loader", "With boot loader") orelse true;
     const with_tools = b.option(bool, "tools", "With tools") orelse true;
+    const firmware_directory = b.option([]const u8, "firmware-directory", "Firmware directory");
 
     const clap = b.dependency("clap", .{});
 
@@ -50,14 +51,32 @@ pub fn build(b: *std.Build) !void {
         });
         tboot_loader.root_module.addImport("linux_headers", linux_headers_module);
 
-        const cpio_tool = b.addRunArtifact(b.addExecutable(.{
+        const cpio_tool = b.addExecutable(.{
             .name = "cpio",
             .target = b.host,
             .root_source_file = b.path("src/cpio/main.zig"),
-        }));
-        cpio_tool.addArtifactArg(tboot_loader);
+        });
+        cpio_tool.root_module.addImport("clap", clap.module("clap"));
+
+        const run_cpio_tool = b.addRunArtifact(cpio_tool);
+        // TODO(jared): Would be nicer to have generic
+        // --file=tboot_loader:/init CLI interface, but don't know how to
+        // obtain path and string format it into that form. Further, would
+        // be nicer to not shell-out to a separate tool at all and just do
+        // the CPIO generation in here.
+        run_cpio_tool.addPrefixedFileArg("-i", tboot_loader.getEmittedBin());
+
+        // TODO(jared): we need any changes in the firmware directory to result in a rebuild
+        if (firmware_directory) |directory| {
+            run_cpio_tool.addPrefixedDirectoryArg("-d", .{ .cwd_relative = directory });
+        }
+
+        const cpio_output_file = run_cpio_tool.addPrefixedOutputFileArg("-o", "tboot-loader.cpio");
+
+        run_cpio_tool.expectExitCode(0);
+
         const cpio_archive = b.addInstallFile(
-            cpio_tool.addOutputFileArg("tboot-loader.cpio"),
+            cpio_output_file,
             "tboot-loader.cpio",
         );
 

pkgs/coreboot/default.nix

@@ -26,16 +26,18 @@ let
 in
 stdenvNoCC.mkDerivation (finalAttrs: {
   pname = "coreboot-${board}";
-  version = "24.08";
+  version = "24.12";
   src =
     (fetchgit {
       url = "https://github.com/coreboot/coreboot";
-      rev = "24.08";
-      hash = "sha256-l+TPBcAsbzYZgsLOPX30ZpgHINAByIIwRHzvjeirIvY=";
+      rev = finalAttrs.version;
+      hash = "sha256-mdxYxE3JiHFDaftNVckeQTVOlF8sWccm74MrpgWtXb4=";
       fetchSubmodules = true;
     }).overrideAttrs
       (_: {
-        # https://github.com/nixos/nixpkgs/blob/4c62505847d88f16df11eff3c81bf9a453a4979e/pkgs/build-support/fetchgit/nix-prefetch-git#L328
+        # Fetch the remaining submodules not fetched by the initial submodule
+        # fetch, since coreboot has `update = none` set on some submodules.
+        # See https://github.com/nixos/nixpkgs/blob/4c62505847d88f16df11eff3c81bf9a453a4979e/pkgs/build-support/fetchgit/nix-prefetch-git#L328
         NIX_PREFETCH_GIT_CHECKOUT_HOOK = ''clean_git -C "$dir" submodule update --init --recursive --checkout -j ''${NIX_BUILD_CORES:-1} --progress'';
       });
   patches = [

src/cpio/main.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const clap = @import("clap");
 
 const ASCII_CPIO_HEADER_SIZE = 110;
 const TRAILER = "TRAILER!!!";
@@ -49,18 +50,13 @@ pub const CpioArchive = struct {
     };
 
     pub fn init(dest: *std.io.StreamSource) !@This() {
-        var cpio = @This(){ .dest = dest };
-
-        // Always start off with a root directory.
-        try cpio.addEntry(null, ".", .Directory, 0o755);
-
-        return cpio;
+        return @This(){ .dest = dest };
     }
 
     pub fn addEntry(
         self: *@This(),
         source: ?*std.io.StreamSource,
-        comptime path: []const u8,
+        path: []const u8,
         entry_type: CpioEntryType,
         perms: u32,
     ) !void {
@@ -82,7 +78,8 @@ pub const CpioArchive = struct {
         };
 
         // add null terminator
-        const filepath = path ++ [_]u8{0};
+        // const filepath = path ++ [_]u8{0};
+        const filepath_len = path.len + 1; // null terminator
 
         // write entry to archive
         {
@@ -97,7 +94,7 @@ pub const CpioArchive = struct {
                 .devminor = 0,
                 .rdevmajor = 0,
                 .rdevminor = 0,
-                .namesize = @intCast(filepath.len),
+                .namesize = @intCast(filepath_len),
             };
 
             try self.dest.writer().print("{X:0>6}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}{X:0>8}", .{
@@ -118,11 +115,12 @@ pub const CpioArchive = struct {
             });
             self.total_written += ASCII_CPIO_HEADER_SIZE;
 
-            try self.dest.writer().writeAll(filepath);
-            self.total_written += filepath.len;
+            try self.dest.writer().writeAll(path);
+            try self.dest.writer().writeByte(0); // null terminator
+            self.total_written += filepath_len;
 
             // pad the file name
-            const header_padding = (4 - ((ASCII_CPIO_HEADER_SIZE + filepath.len) % 4)) % 4;
+            const header_padding = (4 - ((ASCII_CPIO_HEADER_SIZE + filepath_len) % 4)) % 4;
             try self.dest.writer().writeByteNTimes(0, header_padding);
             self.total_written += header_padding;
 
@@ -150,18 +148,18 @@ pub const CpioArchive = struct {
         self.ino += 1;
     }
 
-    pub fn addFile(self: *@This(), comptime path: []const u8, source: *std.io.StreamSource, perms: u32) !void {
+    pub fn addFile(self: *@This(), path: []const u8, source: *std.io.StreamSource, perms: u32) !void {
         try self.addEntry(source, path, .File, perms);
     }
 
-    pub fn addDirectory(self: *@This(), comptime path: []const u8, perms: u32) !void {
+    pub fn addDirectory(self: *@This(), path: []const u8, perms: u32) !void {
         try self.addEntry(null, path, .Directory, perms);
     }
 
     pub fn addSymlink(
         self: *@This(),
-        comptime dstPath: []const u8,
-        comptime srcPath: []const u8,
+        dstPath: []const u8,
+        srcPath: []const u8,
     ) !void {
         var source = std.io.StreamSource{
             .const_buffer = std.io.fixedBufferStream(srcPath),
@@ -171,7 +169,7 @@ pub const CpioArchive = struct {
             &source,
             dstPath,
             .Symlink,
-            0o777, // symlinks always have 777 perms
+            0o777, // make symlinks always have 777 perms
         );
     }
 
@@ -185,12 +183,50 @@ pub const CpioArchive = struct {
 };
 
 pub fn main() !void {
-    var args = std.process.args();
-    _ = args.next().?;
-    const init = args.next().?;
-    const outfile = args.next().?;
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    defer arena.deinit();
+
+    const params = comptime clap.parseParamsComptime(
+        \\-h, --help                    Display this help and exit.
+        \\-i, --init <FILE>             File to add to archive as /init.
+        \\-d, --directory <DIR>...      Directory to add to archive (as-is).
+        \\-o, --output <FILE>           Archive output filepath.
+        \\
+    );
+
+    const parsers = comptime .{
+        .FILE = clap.parsers.string,
+        .DIR = clap.parsers.string,
+    };
+
+    const stderr = std.io.getStdErr().writer();
 
-    var archive_file = try std.fs.cwd().createFile(outfile, .{});
+    var diag = clap.Diagnostic{};
+    var res = clap.parse(clap.Help, &params, &parsers, .{
+        .diagnostic = &diag,
+        .allocator = arena.allocator(),
+    }) catch |err| {
+        try diag.report(stderr, err);
+        try clap.usage(stderr, clap.Help, &params);
+        return;
+    };
+    defer res.deinit();
+
+    if (res.args.help != 0) {
+        return clap.help(std.io.getStdErr().writer(), clap.Help, &params, .{});
+    }
+
+    if (res.args.init == null or res.args.output == null) {
+        try diag.report(stderr, error.InvalidArgument);
+        try clap.usage(std.io.getStdErr().writer(), clap.Help, &params);
+        return;
+    }
+
+    const init: []const u8 = res.args.init.?;
+    const directories: []const []const u8 = res.args.directory;
+    const output: []const u8 = res.args.output.?;
+
+    var archive_file = try std.fs.cwd().createFile(output, .{});
     defer archive_file.close();
 
     var archive_file_source = std.io.StreamSource{ .file = archive_file };
@@ -200,7 +236,74 @@ pub fn main() !void {
     defer init_file.close();
 
     var init_source = std.io.StreamSource{ .file = init_file };
-    try archive.addFile("./init", &init_source, 0o755);
+    try archive.addFile("init", &init_source, 0o755);
+
+    for (directories) |directory_path| {
+        var dir = try std.fs.cwd().openDir(
+            directory_path,
+            .{ .iterate = true },
+        );
+        defer dir.close();
+        try walkDirectory(&arena, directory_path, &archive, dir);
+    }
 
     try archive.finalize();
 }
+
+fn walkDirectory(arena: *std.heap.ArenaAllocator, starting_directory: []const u8, archive: *CpioArchive, directory: std.fs.Dir) !void {
+    var iter = directory.iterate();
+
+    // This buffer is reused across multiple paths.
+    var fullpath_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
+
+    // Before iterating through the directory, first add the directory itself.
+    const full_directory_path = try directory.realpath(".", &fullpath_buf);
+    const directory_path = try std.fs.path.relative(arena.allocator(), starting_directory, full_directory_path);
+
+    // We don't need to add the root directory, as it will already exist.
+    if (!std.mem.eql(u8, directory_path, "")) {
+        try archive.addDirectory(directory_path, 0o755);
+    }
+
+    while (try iter.next()) |dir_entry| {
+        const full_entry_path = try directory.realpath(dir_entry.name, &fullpath_buf);
+        const entry_path = try std.fs.path.relative(arena.allocator(), starting_directory, full_entry_path);
+
+        switch (dir_entry.kind) {
+            .directory => {
+                var sub_directory = try directory.openDir(
+                    dir_entry.name,
+                    .{ .iterate = true },
+                );
+                defer sub_directory.close();
+                try walkDirectory(arena, starting_directory, archive, sub_directory);
+            },
+            .file => {
+                var file = try directory.openFile(dir_entry.name, .{});
+                defer file.close();
+
+                const stat = try file.stat();
+                var source = std.io.StreamSource{ .file = file };
+                try archive.addFile(entry_path, &source, @intCast(stat.mode));
+            },
+            .sym_link => {
+                const resolved_path = try std.fs.path.resolve(arena.allocator(), &.{ starting_directory, entry_path });
+                if (std.mem.startsWith(u8, resolved_path, starting_directory)) {
+                    const symlink_path = try std.fs.path.join(arena.allocator(), &.{ directory_path, dir_entry.name });
+                    try archive.addSymlink(symlink_path, entry_path);
+                } else {
+                    std.log.warn(
+                        "Resolved symlink {s} is outside of {s}, refusing to add to CPIO archive",
+                        .{ resolved_path, starting_directory },
+                    );
+                }
+            },
+            else => |kind| {
+                std.log.warn(
+                    "Do not know how to add file {s} of kind {} to CPIO archive",
+                    .{ full_entry_path, kind },
+                );
+            },
+        }
+    }
+}

src/ymodem.zig

@@ -384,10 +384,10 @@ pub fn main() !void {
     defer arena.deinit();
 
     const params = comptime clap.parseParamsComptime(
-        \\-h, --help           Display this help and exit.
-        \\-t, --tty <FILE>     TTY to send/receive on.
-        \\-d, --dir <DIR>      Directory to send/receive files from/to.
-        \\<ACTION>             send or recv
+        \\-h, --help                 Display this help and exit.
+        \\-t, --tty       <FILE>     TTY to send/receive on.
+        \\-d, --directory <DIR>      Directory to send/receive files from/to.
+        \\<ACTION>                   Action to take ("send" or "recv").
         \\
     );
 
@@ -419,7 +419,7 @@ pub fn main() !void {
         return clap.help(std.io.getStdErr().writer(), clap.Help, &params, .{});
     }
 
-    if (res.positionals.len != 1 or res.args.dir == null or res.args.tty == null) {
+    if (res.positionals.len != 1 or res.args.directory == null or res.args.tty == null) {
         try diag.report(stderr, error.InvalidArgument);
         try clap.usage(std.io.getStdErr().writer(), clap.Help, &params);
         return;
@@ -434,7 +434,7 @@ pub fn main() !void {
     var tty = try system.setupTty(tty_file.handle, .file_transfer);
     defer tty.reset();
 
-    var dir = try std.fs.cwd().openDir(res.args.dir.?, .{});
+    var dir = try std.fs.cwd().openDir(res.args.directory.?, .{});
     defer dir.close();
 
     switch (res.positionals[0]) {

tests/ymodem/default.nix

@@ -31,7 +31,7 @@ testers.runNixOSTest {
           f.write("init=${nodes.machine.system.build.toplevel}/init ${toString nodes.machine.boot.kernelParams}")
 
       def tboot_ymodem(pty):
-          subprocess.run(["${lib.getExe' tinybootTools "tboot-ymodem"}", "send", "--tty", pty, "--dir", host_boot_dir.name])
+          subprocess.run(["${lib.getExe' tinybootTools "tboot-ymodem"}", "send", "--tty", pty, "--directory", host_boot_dir.name])
 
       def lrzsz(pty):
           subprocess.run(f"${lib.getExe' lrzsz "sx"} --ymodem --1k --binary {linux} {initrd} {params} > {pty} < {pty}", shell=True)