feat: Make Rust installation persistent

Cargo.toml

@@ -32,6 +32,7 @@ log = { version = "0.4", default-features = false }
 nix = { version = "0.29.0", default-features = false, features = ["event", "fs", "mount", "sched", "user"] }
 oci-client = { version = "0.14", default-features = false, features = ["rustls-tls"] }
 rand = { version = "0.9", default-features = false, features = ["os_rng", "std_rng"] }
+reqwest = { version = "0.12", default-features = false }
 serde-error = { version = "0.1", default-features = false }
 target-lexicon = { version = "0.12", default-features = false }
 tar = { version = "0.4", default-features = false }

containers/Dockerfile

@@ -82,13 +82,6 @@ COPY package.accept_keywords/* /etc/portage/package.accept_keywords/
 # * Regular expressions:
 #   * libpcre2
 #
-# Install stable and beta Rust toolchains with `default` rustup profile
-# (containing rust-docs, rustfmt, and clippy) for all supported targets.
-#
-# Install nightly Rust toolchains with `complete` rustup profile (containing
-# all components provided by rustup, available only for nightly toolchains)
-# for all supported targets.
-#
 # [0] https://wiki.gentoo.org/wiki/Crossdev
 # [1] https://github.com/llvm/llvm-project/tree/main/llvm-libgcc
 # [2] https://github.com/rust-lang/rust/issues/119504
@@ -165,12 +158,6 @@ RUN emerge-webrsync \
         sys-libs/error-standalone \
         sys-libs/fts-standalone \
         sys-libs/zlib \
-    && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
-    && rustup toolchain install stable beta --profile=default \
-        --target=aarch64-unknown-linux-musl,x86_64-unknown-linux-musl \
-    && rustup toolchain install nightly --profile=complete \
-        --target=aarch64-unknown-linux-musl,x86_64-unknown-linux-musl \
-    && cargo install btfdump \
     && rm -rf \
         /var/cache/binpkgs/* \
         /var/cache/distfiles/* \

src/main.rs

@@ -6,7 +6,7 @@ use std::{
     fmt::Write as _,
     fs,
     io::{BufRead as _, BufReader, Write as _},
-    os::unix::{ffi::OsStrExt as _, process::ExitStatusExt as _},
+    os::unix::{ffi::OsStrExt as _, fs::PermissionsExt as _, process::ExitStatusExt as _},
     path::{Component, Path, PathBuf},
     process::{Command, ExitCode, Stdio},
     str::FromStr,
@@ -16,6 +16,7 @@ use std::{
 
 use anyhow::{anyhow, Context as _};
 use clap::{Parser, Subcommand, ValueEnum};
+use futures::future::try_join;
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use ipc_channel::ipc;
 use log::{debug, error, info};
@@ -241,10 +242,16 @@ struct RunArgs {
 struct ContainerContext {
     /// Indicates whether stdin should be piped to the container.
     interactive: bool,
+    /// Indicates whether rustup has to be bootstraped.
+    bootstrap_rustup: bool,
     /// Path to the main directory with icedragon's state.
     state_dir: PathBuf,
     /// Path to the container root filesystem.
     rootfs_dir: PathBuf,
+    /// Path to the host directory to mount as `.cargo` inside the container.
+    cargo_dir: PathBuf,
+    /// Path to the host directory to mount as `.rustup` inside the container.
+    rustup_dir: PathBuf,
     /// Target triple.
     triple: Triple,
     /// Indicates whether `CC` and `CXX` variables should be set.
@@ -279,17 +286,30 @@ impl ContainerContext {
         override_cc_with_cross: bool,
         volumes: Vec<(PathBuf, PathBuf)>,
         cmd: OsVecDeque,
-    ) -> Self {
+    ) -> anyhow::Result<Self> {
         let rootfs_dir = state_dir.join("rootfs");
-        Self {
+
+        let cargo_dir = state_dir.join("cargo");
+        let rustup_dir = state_dir.join("rustup");
+        let bootstrap_rustup = !cargo_dir
+            .try_exists()
+            .with_context(|| format!("failed to check whether {} exists", cargo_dir.display()))?
+            || !rustup_dir.try_exists().with_context(|| {
+                format!("failed to check whether {} exists", rustup_dir.display())
+            })?;
+
+        Ok(Self {
             interactive,
+            bootstrap_rustup,
             state_dir,
             rootfs_dir,
+            cargo_dir,
+            rustup_dir,
             triple,
             override_cc_with_cross,
             volumes,
             cmd,
-        }
+        })
     }
 }
 
@@ -440,7 +460,7 @@ fn build_container_image(args: BuildContainerImageArgs) -> anyhow::Result<()> {
     if push {
         for tag in &tags {
             if let Err(e) = push_image(&container_engine, tag)
-                .with_context(|| format!("failed to push the tag {tag:?}"))
+                .with_context(|| format!("failed to push the tag {}", tag.display()))
             {
                 errors.push(e);
             }
@@ -556,6 +576,59 @@ fn prepare_env(
     }
 }
 
+/// Downloads a file from the given `url` into `target_file` and shows progress
+/// using `mpb`.
+async fn download_file(
+    mpb: &MultiProgress,
+    url: &str,
+    target_file: &Path,
+) -> anyhow::Result<tokio::fs::File> {
+    let response = reqwest::get(url).await?;
+    let mut response = response.error_for_status()?;
+    let content_length = response
+        .content_length()
+        .ok_or(anyhow!("failed to get the content-length of {url}"))?;
+
+    let mut target_file = tokio::fs::File::create(target_file).await?;
+
+    let pb = mpb.add(ProgressBar::new(content_length));
+    let pb_style =
+        ProgressStyle::with_template(PROGRESS_BAR_TEMPLATE)?.progress_chars(PROGRESS_BAR_CHARS);
+    pb.set_style(pb_style.clone());
+
+    while let Some(chunk) = response.chunk().await? {
+        target_file.write_all(&chunk).await?;
+        pb.inc(chunk.len() as u64);
+    }
+    pb.finish_and_clear();
+
+    Ok(target_file)
+}
+
+/// Downloads the rustup installation script.
+async fn download_rustup(mpb: &MultiProgress, download_dir: &Path) -> anyhow::Result<()> {
+    let rustup_init_file = download_dir.join("rustup-init.sh");
+    let rustup_init_file = download_file(mpb, "https://sh.rustup.rs", &rustup_init_file).await?;
+
+    let mut perms = rustup_init_file.metadata().await?.permissions();
+    perms.set_mode(0o755);
+    rustup_init_file.set_permissions(perms).await?;
+
+    Ok(())
+}
+
+/// Downloads `cargo-binstall`.
+async fn download_cargo_binstall(mpb: &MultiProgress, download_dir: &Path) -> anyhow::Result<()> {
+    let cargo_binstall_file = download_dir.join("cargo-binstall.tgz");
+    download_file(
+        mpb,
+        "https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-musl.full.tgz",
+        &cargo_binstall_file
+    ).await?;
+
+    Ok(())
+}
+
 async fn download_layer(
     client: &OciClient,
     mpb: &MultiProgress,
@@ -615,8 +688,9 @@ async fn download_layer(
     Ok(())
 }
 
-/// Pulls the given `container_image` from the OCI registry.
+/// Pulls the `container_image` from an OCI registry.
 async fn pull_image(
+    ctx: &ContainerContext,
     download_dir: &Path,
     digest_file: &PathBuf,
     container_image: &str,
@@ -668,6 +742,13 @@ async fn pull_image(
         .zip(layer_files.iter())
         .map(|(layer, layer_file)| download_layer(&client, &mpb, &reference, layer, layer_file));
     futures::future::try_join_all(download_layer_futures).await?;
+    if ctx.bootstrap_rustup {
+        try_join(
+            download_rustup(&mpb, download_dir),
+            download_cargo_binstall(&mpb, download_dir),
+        )
+        .await?;
+    }
 
     Ok(Some((digest, layer_files)))
 }
@@ -746,6 +827,32 @@ fn unpack_image(
                 });
             });
         }
+
+        // Move the scripts and binaries, which are related to Rust, into
+        // the `cargo_dir` - which is a directory, which is bind mounted as
+        // `/root/.cargo` inside the container.
+        let cargo_bin_dir = ctx.cargo_dir.join("bin");
+        if ctx.bootstrap_rustup {
+            // `rustup-init.sh` script is used for bootstraping Rustup. We
+            // execute that script inside the container namespaces.
+            s.spawn(|| {
+                fs::copy(
+                    download_dir.join("rustup-init.sh"),
+                    ctx.rootfs_dir.join("rustup-init.sh"),
+                )
+                .unwrap();
+            });
+            // `cargo-binstall`[0] allows to install crates, that publish their
+            // binaries in GitHub releases, without having to build them. We
+            // include it as binary in `{cargo_dir}/.bin` (which ends up as
+            // `/root/.cargo/bin` inside container).
+            //
+            // [0] https://github.com/cargo-bins/cargo-binstall
+            s.spawn(|| {
+                let cargo_binstall_tarball = download_dir.join("cargo-binstall.tgz");
+                unpack_tarball(cargo_binstall_tarball, cargo_bin_dir).unwrap();
+            });
+        }
     });
     for result in &rx {
         result?;
@@ -786,7 +893,12 @@ fn prepare_container(ctx: &ContainerContext, container_image: &str) -> anyhow::R
         .enable_all()
         .build()
         .context("failed to build the tokio runtime for pulling the container image")?
-        .block_on(pull_image(&download_dir, &digest_file, container_image))?
+        .block_on(pull_image(
+            ctx,
+            &download_dir,
+            &digest_file,
+            container_image,
+        ))?
     {
         create_rootfs(&ctx.rootfs_dir)?;
         unpack_image(ctx, &download_dir, layer_files)?;
@@ -797,6 +909,12 @@ fn prepare_container(ctx: &ContainerContext, container_image: &str) -> anyhow::R
             )
         })?;
     }
+
+    if ctx.bootstrap_rustup {
+        fs::create_dir_all(&ctx.cargo_dir)?;
+        fs::create_dir_all(&ctx.rustup_dir)?;
+    }
+
     Ok(())
 }
 
@@ -993,6 +1111,9 @@ fn mount_volumes(ctx: &ContainerContext) -> anyhow::Result<()> {
     if let Some(ssh_auth_sock) = env::var_os("SSH_AUTH_SOCK") {
         bind_mount(&ctx.rootfs_dir, &ssh_auth_sock, &ssh_auth_sock)?;
     }
+    // Mount Rust toolchain.
+    bind_mount(&ctx.rootfs_dir, &ctx.cargo_dir, "/root/.cargo")?;
+    bind_mount(&ctx.rootfs_dir, &ctx.rustup_dir, "/root/.rustup")?;
     // Mount all the user-provided volumes.
     for (src, dst) in &ctx.volumes {
         bind_mount(&ctx.rootfs_dir, src, dst)
@@ -1001,12 +1122,75 @@ fn mount_volumes(ctx: &ContainerContext) -> anyhow::Result<()> {
     Ok(())
 }
 
+fn run_bootstrap_command<P, I, S>(
+    program: P,
+    args: I,
+    envs: &[(Cow<'static, OsStr>, Cow<'static, OsStr>)],
+) -> anyhow::Result<()>
+where
+    P: AsRef<OsStr>,
+    I: IntoIterator<Item = S>,
+    S: AsRef<OsStr>,
+{
+    let mut cmd = Command::new(program);
+    cmd.args(args).env_clear().envs(envs.iter().map(
+        // We call `to_owned` only to make the `&(_, _)` tuples owned. This
+        // doesn't make any actual copies of data carried in `Cow`
+        // instances.
+        ToOwned::to_owned,
+    ));
+    if !cmd.spawn()?.wait()?.success() {
+        return Err(anyhow!("failed to execute the command {cmd:?}"));
+    }
+    Ok(())
+}
+
+fn bootstrap_rustup(envs: &[(Cow<'static, OsStr>, Cow<'static, OsStr>)]) -> anyhow::Result<()> {
+    run_bootstrap_command("/bin/sh", ["/rustup-init.sh", "-y"], envs)
+        .context("failed to bootstrap rustup")?;
+    // Install stable and beta Rust toolchains with `default` rustup
+    // profile (containing rust-docs, rustfmt, and clippy) for all
+    // supported targets.
+    run_bootstrap_command(
+        "rustup",
+        [
+            "toolchain",
+            "install",
+            "stable",
+            "beta",
+            "--profile=default",
+            "--target=aarch64-unknown-linux-musl,x86_64-unknown-linux-musl",
+        ],
+        envs,
+    )
+    .context("failed to install stable and beta Rust toolchains")?;
+    // Install nightly Rust toolchains with `complete` rustup profile
+    // (containing all components provided by rustup, available only for
+    // nightly toolchains) for all supported targets.
+    run_bootstrap_command(
+        "rustup",
+        [
+            "toolchain",
+            "install",
+            "nightly",
+            "--profile=complete",
+            "--target=aarch64-unknown-linux-musl,x86_64-unknown-linux-musl",
+        ],
+        envs,
+    )
+    .context("failed to install nightly Rust toolchain")
+}
+
 fn container_child(ctx: &ContainerContext) -> anyhow::Result<u8> {
     mount_volumes(ctx)?;
     chroot(&ctx.rootfs_dir).context("`chroot` syscall failed")?;
     chdir("/src").context("failed to change directory to `/src`")?;
 
-    let envs = prepare_env(&ctx.triple, ctx.override_cc_with_cross);
+    let envs: Vec<_> = prepare_env(&ctx.triple, ctx.override_cc_with_cross).collect();
+
+    if ctx.bootstrap_rustup {
+        bootstrap_rustup(&envs)?;
+    }
 
     let mut cmd = ctx.cmd.command()?;
     cmd.stdout(Stdio::inherit()).stderr(Stdio::inherit());
@@ -1254,7 +1438,7 @@ fn run(
         override_cc_with_cross,
         volumes,
         cmd,
-    );
+    )?;
     prepare_container(&ctx, container_image)?;
     run_container(ctx)
 }

tests/hello-world/Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "hello-world"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]

tests/hello-world/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

tests/tests.rs

@@ -135,6 +135,19 @@ where
     assert_eq!(elf.header.e_machine, elf_machine);
 }
 
+/// Tests whether binaries installed through `cargo install` are stored
+/// persistently.
+#[test]
+fn test_cargo_install() {
+    let hello_world_dir = env::current_dir().unwrap().join("tests/hello-world");
+    icedragon_cmd(&hello_world_dir, "cargo", None, &[])
+        .args(["install", "--path", "."])
+        .assert_success();
+    icedragon_cmd(&hello_world_dir, "run", None, &[])
+        .args(["hello-world", "--version"])
+        .assert_success();
+}
+
 /// Tests cargo support by cross-compiling pulsar.
 #[test_case("aarch64-unknown-linux-musl", elf_header::EM_AARCH64 ; "aarch64")]
 #[test_case("x86_64-unknown-linux-musl", elf_header::EM_X86_64 ; "x86_64")]