Make macOS dpd mgd build from source workaround official (#9312)

Dendrite and maghemite do not build mac binaries for `dpd`, `swadm`, and
`mgd`. To work around this, I wrote a
[script](https://gist.github.com/david-crespo/797f955fa116a1ebbb720ea3513e6f29)
I've been passing around to all macOS users in secret. This script
checks out the right commit locally and builds those binaries from
source. @davepacheco suggested this could just be part of the normal
install xtask. Claude got it done in about 5 minutes and it appears to
work quite well.

Differences from the original script to not assume anything about your
computer:

1. Commit-hash based binary caching is portable — cache lives in
`out/.build-cache/{project}/{commit}/`
1. Clone dendrite and maghemite from scratch in temp dirs instead of
assuming they are already cloned next to omicron

Author: david-crespo

Cargo.lock

@@ -21,4 +21,5 @@ slog-async.workspace = true
 slog-term.workspace = true
 strum.workspace = true
 tar.workspace = true
+tempfile.workspace = true
 tokio.workspace = true

dev-tools/downloader/Cargo.toml

@@ -216,6 +216,114 @@ impl<'a> Downloader<'a> {
     ) -> Self {
         Self { log, output_dir, versions_dir }
     }
+
+    /// Build a binary from a git repository at a specific commit.
+    ///
+    /// This function:
+    /// 1. Checks for cached binaries at `out/.build-cache/{project}/{commit}/`
+    /// 2. If not cached, shallow clones the repo to a temp directory
+    /// 3. Builds the specified binaries with cargo
+    /// 4. Caches the built binaries
+    /// 5. Returns paths to the cached binaries
+    async fn build_from_git(
+        &self,
+        project: &str,
+        commit: &str,
+        binaries: &[(&str, &[&str])], // (binary_name, cargo_args)
+    ) -> Result<Vec<Utf8PathBuf>> {
+        let cache_dir =
+            self.output_dir.join(".build-cache").join(project).join(commit);
+
+        // Check if all binaries are already cached
+        let mut cached_paths = Vec::new();
+        let mut all_cached = true;
+        for (binary_name, _) in binaries {
+            let cached_path = cache_dir.join(binary_name);
+            if !cached_path.exists() {
+                all_cached = false;
+                break;
+            }
+            cached_paths.push(cached_path);
+        }
+
+        if all_cached {
+            info!(self.log, "Found cached binaries for {project} at {commit}"; "cache_dir" => %cache_dir);
+            return Ok(cached_paths);
+        }
+
+        // Need to build - create temp directory
+        info!(self.log, "Building {project} from source at commit {commit}");
+        let temp_dir = tempfile::tempdir()?;
+        let temp_path = Utf8PathBuf::try_from(temp_dir.path().to_path_buf())?;
+
+        // Clone and checkout the specific commit
+        let repo_url = format!("https://github.com/oxidecomputer/{}", project);
+        info!(self.log, "Cloning {repo_url}");
+        let mut clone_cmd = Command::new("git");
+        clone_cmd
+            .arg("clone")
+            .arg("--filter=blob:none")
+            .arg(&repo_url)
+            .arg(&temp_path);
+
+        let clone_output = clone_cmd.output().await?;
+        if !clone_output.status.success() {
+            let stderr = String::from_utf8_lossy(&clone_output.stderr);
+            bail!("Failed to clone {repo_url}: {stderr}");
+        }
+
+        // Checkout the specific commit
+        info!(self.log, "Checking out commit {commit}");
+        let mut checkout_cmd = Command::new("git");
+        checkout_cmd.arg("checkout").arg(commit).current_dir(&temp_path);
+
+        let checkout_output = checkout_cmd.output().await?;
+        if !checkout_output.status.success() {
+            let stderr = String::from_utf8_lossy(&checkout_output.stderr);
+            bail!("Failed to checkout {commit}: {stderr}");
+        }
+
+        // Build each binary
+        tokio::fs::create_dir_all(&cache_dir).await?;
+        let mut result_paths = Vec::new();
+
+        for (binary_name, cargo_args) in binaries {
+            info!(self.log, "Building {binary_name}"; "args" => ?cargo_args);
+
+            let mut build_cmd = Command::new("cargo");
+            build_cmd
+                .arg("build")
+                .arg("--release")
+                .arg("--bin")
+                .arg(binary_name)
+                .args(*cargo_args)
+                .current_dir(&temp_path);
+
+            let build_output = build_cmd.output().await?;
+            if !build_output.status.success() {
+                let stderr = String::from_utf8_lossy(&build_output.stderr);
+                bail!("Failed to build {binary_name}: {stderr}");
+            }
+
+            // Always build in release mode
+            let source_path =
+                temp_path.join("target").join("release").join(binary_name);
+
+            if !source_path.exists() {
+                bail!("Expected binary not found at {source_path}");
+            }
+
+            // Copy to cache
+            let cached_path = cache_dir.join(binary_name);
+            tokio::fs::copy(&source_path, &cached_path).await?;
+            set_permissions(&cached_path, 0o755).await?;
+
+            result_paths.push(cached_path);
+        }
+
+        info!(self.log, "Successfully built and cached {project} binaries");
+        Ok(result_paths)
+    }
 }
 
 /// Parses a file of the format:
@@ -738,13 +846,24 @@ impl Downloader<'_> {
             }
             Os::Illumos => {}
             Os::Mac => {
-                warn!(self.log, "WARNING: Dendrite not available for Mac");
-                warn!(self.log, "Network APIs will be unavailable");
-
-                let path = bin_dir.join("dpd");
-                tokio::fs::write(&path, "echo 'unsupported os' && exit 1")
-                    .await?;
-                set_permissions(&path, 0o755).await?;
+                info!(self.log, "Building dendrite from source for macOS");
+
+                let binaries = [
+                    ("dpd", &["--features=tofino_stub"][..]),
+                    ("swadm", &[][..]),
+                ];
+
+                let built_binaries =
+                    self.build_from_git("dendrite", &commit, &binaries).await?;
+
+                // Copy built binaries to bin_dir
+                for (binary_path, (binary_name, _)) in
+                    built_binaries.iter().zip(binaries.iter())
+                {
+                    let dest = bin_dir.join(binary_name);
+                    tokio::fs::copy(binary_path, &dest).await?;
+                    set_permissions(&dest, 0o755).await?;
+                }
             }
         }
 
@@ -807,7 +926,21 @@ impl Downloader<'_> {
                 set_permissions(&path, 0o755).await?;
                 tokio::fs::copy(path, binary_dir.join(filename)).await?;
             }
-            _ => (),
+            Os::Mac => {
+                info!(self.log, "Building maghemite from source for macOS");
+
+                let binaries = [("mgd", &["--no-default-features"][..])];
+
+                let built_binaries = self
+                    .build_from_git("maghemite", &commit, &binaries)
+                    .await?;
+
+                // Copy built binary to binary_dir
+                let dest = binary_dir.join("mgd");
+                tokio::fs::copy(&built_binaries[0], &dest).await?;
+                set_permissions(&dest, 0o755).await?;
+            }
+            Os::Illumos => (),
         }
 
         Ok(())

dev-tools/downloader/src/lib.rs

@@ -199,7 +199,7 @@ function install_packages {
     )
     confirm "Install (or update) [${packages[*]}]?" && brew install "${packages[@]}"
   else
-    echo "Unsupported OS: ${HOST_OS}"
+    echo "Skipping builder prereqs for unsupported OS: ${HOST_OS}"
     exit 1
   fi
 }

tools/install_builder_prerequisites.sh

@@ -153,7 +153,7 @@ function install_packages {
       confirm "Install (or update) [${packages[*]}]?" && $maybe_sudo apt-get install "${packages[@]}"
     fi
   else
-    echo "Unsupported OS: ${HOST_OS}"
+    echo "Skipping runner prereqs for unsupported OS: ${HOST_OS}"
     exit 1
   fi
 }