Add a CPU limit, raise example memory limit

The combination of no job limit and a low memory limit has been causing
a lot of OOMs in crater runs. This adds a job limit and enforces it
including for doctests, which are built by the test runner. If nothing
else, setting this CPU limit provides reproducible OOMs because many
crates have peak memory usage proportional to build parallelism.

Author: saethlin

Cargo.lock

@@ -64,6 +64,7 @@ ctrlc = "3.1.3"
 prometheus = "0.7.0"
 cargo_metadata = "0.12.1"
 indexmap = "1.4.0"
+git2 = { version = "0.13", features = ["vendored-libgit2"] }
 
 [dev-dependencies]
 assert_cmd = "1.0.2"

Cargo.toml

@@ -27,7 +27,8 @@ local-crates = []
 
 [sandbox]
 # Maximum amount of RAM allowed during builds
-memory-limit = "1536M"  # 1.5G
+memory-limit = "4096M"  # 4G, 1G/cpu
+cpu-limit = 4
 # Restrictions on the amount of information stored in build logs
 build-log-max-size = "5M"
 build-log-max-lines = 10000

config.toml

@@ -74,6 +74,7 @@ pub struct DemoCrates {
 #[serde(rename_all = "kebab-case")]
 pub struct SandboxConfig {
     pub memory_limit: Size,
+    pub cpu_limit: u16,
     pub build_log_max_size: Size,
     pub build_log_max_lines: usize,
 }
@@ -256,6 +257,7 @@ impl Default for Config {
             local_crates: HashMap::new(),
             sandbox: SandboxConfig {
                 memory_limit: Size::Gigabytes(2),
+                cpu_limit: 1,
                 build_log_max_size: Size::Megabytes(1),
                 build_log_max_lines: 1000,
             },
@@ -308,6 +310,7 @@ mod tests {
             "local-crates = []\n",
             "[sandbox]\n",
             "memory-limit = \"2G\"\n",
+            "cpu-limit = 1\n",
             "build-log-max-size = \"2M\"\n",
             "build-log-max-lines = 1000\n",
             "[crates]\n",

src/config.rs

@@ -89,6 +89,9 @@ fn run_cargo<DB: WriteResults>(
         rustflags.push(' ');
         rustflags.push_str(tc_rustflags);
     }
+    // Limit debuginfo in an effort to decrease linker peak memory usage, while retaining enough
+    // information to get useful backtraces.
+    rustflags.push_str(" -Cdebuginfo=1");
 
     let rustflags_env = if let Some(&"doc") = args.get(0) {
         "RUSTDOCFLAGS"
@@ -210,6 +213,7 @@ pub(super) fn run_test<DB: WriteResults>(
                 );
                 let sandbox = SandboxBuilder::new()
                     .memory_limit(Some(ctx.config.sandbox.memory_limit.to_bytes()))
+                    .cpu_limit(Some(f32::from(ctx.config.sandbox.cpu_limit)))
                     .enable_networking(false);
 
                 let krate = &ctx.krate.to_rustwide();
@@ -235,28 +239,51 @@ fn build<DB: WriteResults>(
     build_env: &Build,
     local_packages_id: &HashSet<PackageId>,
 ) -> Fallible<()> {
+    let cpus = ctx.config.sandbox.cpu_limit.to_string();
     run_cargo(
         ctx,
         build_env,
-        &["build", "--frozen", "--message-format=json"],
+        &[
+            "build",
+            "--frozen",
+            "--message-format=json",
+            "--jobs",
+            &cpus,
+        ],
         true,
         local_packages_id,
     )?;
     run_cargo(
         ctx,
         build_env,
-        &["test", "--frozen", "--no-run", "--message-format=json"],
+        &[
+            "test",
+            "--frozen",
+            "--no-run",
+            "--message-format=json",
+            "--jobs",
+            &cpus,
+        ],
         true,
         local_packages_id,
     )?;
     Ok(())
 }
 
 fn test<DB: WriteResults>(ctx: &TaskCtx<DB>, build_env: &Build) -> Fallible<()> {
+    let cpus = ctx.config.sandbox.cpu_limit.to_string();
     run_cargo(
         ctx,
         build_env,
-        &["test", "--frozen"],
+        &[
+            "test",
+            "--frozen",
+            "--jobs",
+            &cpus,
+            "--",
+            "--test-threads",
+            &cpus,
+        ],
         false,
         &HashSet::new(),
     )
@@ -299,6 +326,7 @@ pub(super) fn test_check_only<DB: WriteResults>(
     build_env: &Build,
     local_packages_id: &HashSet<PackageId>,
 ) -> Fallible<TestResult> {
+    let cpus = ctx.config.sandbox.cpu_limit.to_string();
     if let Err(err) = run_cargo(
         ctx,
         build_env,
@@ -308,6 +336,8 @@ pub(super) fn test_check_only<DB: WriteResults>(
             "--all",
             "--all-targets",
             "--message-format=json",
+            "--jobs",
+            &cpus,
         ],
         true,
         local_packages_id,
@@ -323,6 +353,7 @@ pub(super) fn test_clippy_only<DB: WriteResults>(
     build_env: &Build,
     local_packages_id: &HashSet<PackageId>,
 ) -> Fallible<TestResult> {
+    let cpus = ctx.config.sandbox.cpu_limit.to_string();
     if let Err(err) = run_cargo(
         ctx,
         build_env,
@@ -332,6 +363,8 @@ pub(super) fn test_clippy_only<DB: WriteResults>(
             "--all",
             "--all-targets",
             "--message-format=json",
+            "--jobs",
+            &cpus,
         ],
         true,
         local_packages_id,
@@ -347,6 +380,7 @@ pub(super) fn test_rustdoc<DB: WriteResults>(
     build_env: &Build,
     local_packages_id: &HashSet<PackageId>,
 ) -> Fallible<TestResult> {
+    let cpus = ctx.config.sandbox.cpu_limit.to_string();
     let res = run_cargo(
         ctx,
         build_env,
@@ -356,6 +390,8 @@ pub(super) fn test_rustdoc<DB: WriteResults>(
             "--no-deps",
             "--document-private-items",
             "--message-format=json",
+            "--jobs",
+            &cpus,
         ],
         true,
         local_packages_id,

src/runner/test.rs

@@ -17,6 +17,7 @@ local-crates = []
 
 [sandbox]
 memory-limit = "1536M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/check_config/bad-duplicate-crate.toml

@@ -17,6 +17,7 @@ local-crates = []
 
 [sandbox]
 memory-limit = "1536M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/check_config/bad-missing-crate.toml

@@ -17,6 +17,7 @@ local-crates = []
 
 [sandbox]
 memory-limit = "1536M"
+cpu-limit= 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/check_config/bad-missing-repo.toml

@@ -17,6 +17,7 @@ local-crates = []
 
 [sandbox]
 memory-limit = "1536M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/check_config/good.toml

@@ -18,6 +18,7 @@ local-crates = ["build-pass", "build-fail", "test-fail"]
 
 [sandbox]
 memory-limit = "512M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/minicrater/blacklist/config.toml

@@ -17,6 +17,7 @@ local-crates = ["build-pass", "clippy-warn"]
 
 [sandbox]
 memory-limit = "512M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/minicrater/clippy/config.toml

@@ -17,6 +17,7 @@ local-crates = []
 
 [sandbox]
 memory-limit = "512M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/minicrater/full/config.toml

@@ -17,6 +17,7 @@ local-crates = ["build-pass", "build-fail", "test-fail"]
 
 [sandbox]
 memory-limit = "512M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/minicrater/ignore-blacklist/config.toml

@@ -17,6 +17,7 @@ local-crates = []
 
 [sandbox]
 memory-limit = "512M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/minicrater/missing-repo/config.toml

@@ -17,6 +17,7 @@ local-crates = ["build-pass", "memory-hungry"]
 
 [sandbox]
 memory-limit = "512M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000
 

tests/minicrater/resource-exhaustion/config.toml

@@ -17,6 +17,7 @@ local-crates = ["build-pass", "beta-regression"]
 
 [sandbox]
 memory-limit = "512M"
+cpu-limit = 1
 build-log-max-size = "2M"
 build-log-max-lines = 1000