Auto merge of #105609 - bjorn3:shrink_rustc_dev, r=jyn514

Only include metadata for non-dynamic libraries in rustc-dev

The actual object code should be linked from librustc_driver.so, which is still included in rustc-dev. This saves on download time and disk usage.

Fixes #103538

Author: bors

compiler/rustc_error_messages/locales/en-US/metadata.ftl

@@ -4,6 +4,11 @@ metadata_rlib_required =
 metadata_lib_required =
     crate `{$crate_name}` required to be available in {$kind} format, but was not found in this form
 
+metadata_rustc_lib_required =
+    crate `{$crate_name}` required to be available in {$kind} format, but was not found in this form
+    .note = only .rmeta files are distributed for `rustc_private` crates other than `rustc_driver`
+    .help = try adding `extern crate rustc_driver;` at the top level of this crate
+
 metadata_crate_dep_multiple =
     cannot satisfy dependencies so `{$crate_name}` only shows up once
     .help = having upstream crates all available in one format will likely make this go away

compiler/rustc_metadata/src/dependency_format.rs

@@ -54,7 +54,7 @@
 use crate::creader::CStore;
 use crate::errors::{
     BadPanicStrategy, CrateDepMultiple, IncompatiblePanicInDropStrategy, LibRequired,
-    RequiredPanicStrategy, RlibRequired, TwoPanicRuntimes,
+    RequiredPanicStrategy, RlibRequired, RustcLibRequired, TwoPanicRuntimes,
 };
 
 use rustc_data_structures::fx::FxHashMap;
@@ -224,7 +224,12 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
                     Linkage::Static => "rlib",
                     _ => "dylib",
                 };
-                sess.emit_err(LibRequired { crate_name: tcx.crate_name(cnum), kind: kind });
+                let crate_name = tcx.crate_name(cnum);
+                if crate_name.as_str().starts_with("rustc_") {
+                    sess.emit_err(RustcLibRequired { crate_name, kind });
+                } else {
+                    sess.emit_err(LibRequired { crate_name, kind });
+                }
             }
         }
     }

compiler/rustc_metadata/src/errors.rs

@@ -24,6 +24,14 @@ pub struct LibRequired<'a> {
     pub kind: &'a str,
 }
 
+#[derive(Diagnostic)]
+#[diag(metadata_rustc_lib_required)]
+#[help]
+pub struct RustcLibRequired<'a> {
+    pub crate_name: Symbol,
+    pub kind: &'a str,
+}
+
 #[derive(Diagnostic)]
 #[diag(metadata_crate_dep_multiple)]
 #[help]

src/bootstrap/check.rs

@@ -105,7 +105,7 @@ impl Step for Std {
             "Checking stage{} library artifacts ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), vec![], true, false);
 
         // We skip populating the sysroot in non-zero stage because that'll lead
         // to rlib/rmeta conflicts if std gets built during this session.
@@ -155,7 +155,14 @@ impl Step for Std {
             "Checking stage{} library test/bench/example targets ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &libstd_test_stamp(builder, compiler, target), vec![], true);
+        run_cargo(
+            builder,
+            cargo,
+            &libstd_test_stamp(builder, compiler, target),
+            vec![],
+            true,
+            false,
+        );
     }
 }
 
@@ -225,7 +232,7 @@ impl Step for Rustc {
             "Checking stage{} compiler artifacts ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], true, false);
 
         let libdir = builder.sysroot_libdir(compiler, target);
         let hostdir = builder.sysroot_libdir(compiler, compiler.host);
@@ -285,6 +292,7 @@ impl Step for CodegenBackend {
             &codegen_backend_stamp(builder, compiler, target, backend),
             vec![],
             true,
+            false,
         );
     }
 }
@@ -343,7 +351,7 @@ impl Step for RustAnalyzer {
             "Checking stage{} {} artifacts ({} -> {})",
             compiler.stage, "rust-analyzer", &compiler.host.triple, target.triple
         ));
-        run_cargo(builder, cargo, &stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &stamp(builder, compiler, target), vec![], true, false);
 
         /// Cargo's output path in a given stage, compiled by a particular
         /// compiler for the specified target.
@@ -417,6 +425,7 @@ macro_rules! tool_check_step {
                     &stamp(builder, compiler, target),
                     vec![],
                     true,
+                    false,
                 );
 
                 /// Cargo's output path in a given stage, compiled by a particular

src/bootstrap/compile.rs

@@ -141,7 +141,14 @@ impl Step for Std {
             &compiler.host,
             target,
         ));
-        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), target_deps, false);
+        run_cargo(
+            builder,
+            cargo,
+            &libstd_stamp(builder, compiler, target),
+            target_deps,
+            false,
+            false,
+        );
 
         builder.ensure(StdLink::from_std(
             self,
@@ -728,7 +735,14 @@ impl Step for Rustc {
             &compiler.host,
             target,
         ));
-        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], false);
+        run_cargo(
+            builder,
+            cargo,
+            &librustc_stamp(builder, compiler, target),
+            vec![],
+            false,
+            true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files.
+        );
 
         builder.ensure(RustcLink::from_rustc(
             self,
@@ -984,7 +998,7 @@ impl Step for CodegenBackend {
             "Building stage{} codegen backend {} ({} -> {})",
             compiler.stage, backend, &compiler.host, target
         ));
-        let files = run_cargo(builder, cargo, &tmp_stamp, vec![], false);
+        let files = run_cargo(builder, cargo, &tmp_stamp, vec![], false, false);
         if builder.config.dry_run() {
             return;
         }
@@ -1411,6 +1425,7 @@ pub fn run_cargo(
     stamp: &Path,
     additional_target_deps: Vec<(PathBuf, DependencyType)>,
     is_check: bool,
+    rlib_only_metadata: bool,
 ) -> Vec<PathBuf> {
     if builder.config.dry_run() {
         return Vec::new();
@@ -1444,13 +1459,35 @@ pub fn run_cargo(
         };
         for filename in filenames {
             // Skip files like executables
-            if !(filename.ends_with(".rlib")
-                || filename.ends_with(".lib")
+            let mut keep = false;
+            if filename.ends_with(".lib")
                 || filename.ends_with(".a")
                 || is_debug_info(&filename)
                 || is_dylib(&filename)
-                || (is_check && filename.ends_with(".rmeta")))
             {
+                // Always keep native libraries, rust dylibs and debuginfo
+                keep = true;
+            }
+            if is_check && filename.ends_with(".rmeta") {
+                // During check builds we need to keep crate metadata
+                keep = true;
+            } else if rlib_only_metadata {
+                if filename.contains("jemalloc_sys") || filename.contains("rustc_smir") {
+                    // jemalloc_sys and rustc_smir are not linked into librustc_driver.so,
+                    // so we need to distribute them as rlib to be able to use them.
+                    keep |= filename.ends_with(".rlib");
+                } else {
+                    // Distribute the rest of the rustc crates as rmeta files only to reduce
+                    // the tarball sizes by about 50%. The object files are linked into
+                    // librustc_driver.so, so it is still possible to link against them.
+                    keep |= filename.ends_with(".rmeta");
+                }
+            } else {
+                // In all other cases keep all rlibs
+                keep |= filename.ends_with(".rlib");
+            }
+
+            if !keep {
                 continue;
             }
 

src/test/run-make-fulldeps/save-analysis/foo.rs

@@ -5,6 +5,11 @@
 extern crate rustc_graphviz;
 // A simple rust project
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 extern crate krate2;
 extern crate krate2 as krate3;
 

src/test/ui-fulldeps/deriving-encodable-decodable-box.rs

@@ -6,6 +6,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 use rustc_serialize::opaque::{MemDecoder, MemEncoder};
 use rustc_serialize::{Decodable, Encodable, Encoder};

src/test/ui-fulldeps/deriving-encodable-decodable-cell-refcell.rs

@@ -8,6 +8,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 use rustc_serialize::opaque::{MemDecoder, MemEncoder};
 use rustc_serialize::{Decodable, Encodable, Encoder};

src/test/ui-fulldeps/deriving-global.rs

@@ -5,6 +5,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 mod submod {
     use rustc_macros::{Decodable, Encodable};
 

src/test/ui-fulldeps/deriving-hygiene.rs

@@ -7,6 +7,11 @@ extern crate rustc_serialize;
 
 use rustc_macros::{Decodable, Encodable};
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 pub const other: u8 = 1;
 pub const f: u8 = 1;
 pub const d: u8 = 1;

src/test/ui-fulldeps/dropck_tarena_sound_drop.rs

@@ -14,6 +14,11 @@
 
 extern crate rustc_arena;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_arena::TypedArena;
 
 trait HasId { fn count(&self) -> usize; }

src/test/ui-fulldeps/empty-struct-braces-derive.rs

@@ -6,6 +6,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Debug, Encodable, Decodable)]

src/test/ui-fulldeps/issue-14021.rs

@@ -7,6 +7,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 use rustc_serialize::opaque::{MemDecoder, MemEncoder};
 use rustc_serialize::{Decodable, Encodable, Encoder};

src/test/ui-fulldeps/missing-rustc-driver-error.rs

@@ -0,0 +1,11 @@
+// Test that we get the following hint when trying to use a compiler crate without rustc_driver.
+// error-pattern: try adding `extern crate rustc_driver;` at the top level of this crate
+// compile-flags: --emit link
+// The exactly list of required crates depends on the target. as such only test Unix targets.
+// only-unix
+
+#![feature(rustc_private)]
+
+extern crate rustc_serialize;
+
+fn main() {}

src/test/ui-fulldeps/missing-rustc-driver-error.stderr

@@ -0,0 +1,24 @@
+error: crate `rustc_serialize` required to be available in rlib format, but was not found in this form
+   |
+   = help: try adding `extern crate rustc_driver;` at the top level of this crate
+
+error: crate `smallvec` required to be available in rlib format, but was not found in this form
+
+error: crate `thin_vec` required to be available in rlib format, but was not found in this form
+
+error: crate `indexmap` required to be available in rlib format, but was not found in this form
+
+error: crate `hashbrown` required to be available in rlib format, but was not found in this form
+
+error: crate `ahash` required to be available in rlib format, but was not found in this form
+
+error: crate `once_cell` required to be available in rlib format, but was not found in this form
+
+error: crate `getrandom` required to be available in rlib format, but was not found in this form
+
+error: crate `cfg_if` required to be available in rlib format, but was not found in this form
+
+error: crate `libc` required to be available in rlib format, but was not found in this form
+
+error: aborting due to 10 previous errors
+

src/test/ui-fulldeps/mod_dir_path_canonicalized.rs

@@ -10,6 +10,11 @@ extern crate rustc_parse;
 extern crate rustc_session;
 extern crate rustc_span;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_parse::new_parser_from_file;
 use rustc_session::parse::ParseSess;
 use rustc_span::source_map::FilePathMapping;

src/test/ui-fulldeps/pprust-expr-roundtrip.rs

@@ -27,6 +27,11 @@ extern crate rustc_session;
 extern crate rustc_span;
 extern crate thin_vec;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_ast::mut_visit::{self, visit_clobber, MutVisitor};
 use rustc_ast::ptr::P;
 use rustc_ast::*;

src/test/ui-fulldeps/regions-mock-tcx.rs

@@ -14,6 +14,11 @@
 extern crate rustc_arena;
 extern crate libc;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use TypeStructure::{TypeInt, TypeFunction};
 use AstKind::{ExprInt, ExprVar, ExprLambda};
 use rustc_arena::TypedArena;

src/test/ui-fulldeps/rustc_encodable_hygiene.rs

@@ -6,6 +6,11 @@ extern crate rustc_macros;
 #[allow(dead_code)]
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 
 #[derive(Decodable, Encodable, Debug)]

src/tools/miri/src/lib.rs

@@ -54,6 +54,11 @@ extern crate rustc_session;
 extern crate rustc_span;
 extern crate rustc_target;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 mod borrow_tracker;
 mod clock;
 mod concurrency;