Auto merge of rust-lang#136779 - jieyouxu:rahhh, r=<try>

[DO NOT MERGE] `./x test rust-analyzer`

I somehow made `./x test rust-analyzer` work on my machine[^machine], **but at what cost?**

Not intended for merge but only as a reference. If we do want to land this, I'll need to tidy this up.

Notes:

- I abused a bunch of cargo features  `in-rust-tree`. It probably doesn't need to be, and simply `--cfg` might work. I was trying to get the main rust-analyzer tests to build *at all*. Anything building is already a miracle.
- I had to slap a bunch of the following to all the r-a crates to get the tests to build at all. I don't 100% understand why, but otherwise I get a whole ton of ``expected `rustc_lexer` to be available in rlib format`` build failures.

```rs
#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
#[cfg(all(feature = "in-rust-tree", test))]
extern crate rustc_driver as _;
```

- Skipped one config test that was fixed on r-a master but not synced here in r-l/r yet.

[^machine]: `x86_64-unknown-linux-gnu`, haven't bothered trying this on msvc yet.

try-job: aarch64-gnu
try-job: x86_64-apple-1
try-job: aarch64-apple
try-job: i686-mingw
try-job: x86_64-mingw-1
try-job: i686-msvc-1
try-job: x86_64-msvc-1

src/bootstrap/src/core/build_steps/test.rs

@@ -9,6 +9,8 @@ use std::path::{Path, PathBuf};
 use std::{env, fs, iter};
 
 use clap_complete::shells;
+#[cfg(feature = "tracing")]
+use tracing::instrument;
 
 use crate::core::build_steps::compile::run_cargo;
 use crate::core::build_steps::doc::DocumentationFormat;
@@ -28,7 +30,7 @@ use crate::utils::helpers::{
     linker_flags, t, target_supports_cranelift_backend, up_to_date,
 };
 use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests};
-use crate::{CLang, DocTests, GitRepo, Mode, PathSet, envify};
+use crate::{CLang, DocTests, GitRepo, Mode, PathSet, debug, envify};
 
 const ADB_TEST_DIR: &str = "/data/local/tmp/work";
 
@@ -354,50 +356,301 @@ impl Step for RustAnalyzer {
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/tools/rust-analyzer")
+        run.path("src/tools/rust-analyzer").alias("rust-analyzer")
     }
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target });
     }
 
-    /// Runs `cargo test` for rust-analyzer
+    #[cfg_attr(
+        feature = "tracing",
+        instrument(
+            level = "debug",
+            name = "RustAnalyzer::run",
+            skip_all,
+            fields(stage = self.stage, host = ?self.host),
+        ),
+    )]
+    fn run(self, builder: &Builder<'_>) {
+        debug!(stage = self.stage, host = ?self.host, "ensuring compiler");
+        let compiler = builder.compiler(self.stage, self.host);
+
+        debug!(stage = self.stage, host = ?self.host, "ensuring std");
+        builder.ensure(compile::Rustc::new(compiler, self.host));
+
+        let mut cargo = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolRustc,
+            self.host,
+            Kind::Test,
+            "src/tools/rust-analyzer",
+            SourceType::InTree,
+            &["in-rust-tree".to_owned()],
+        );
+        cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES);
+
+        // RA's test suite tries to write to the source directory, that can't work in Rust CI.
+        cargo.env("SKIP_SLOW_TESTS", "1");
+
+        // NOTE: unlike `proc-macro-srv` step, we must **not** set `CARGO_WORKSPACE_DIR` because I
+        // believe `src/tools/rust-analyzer/.cargo/config.toml`'s relative `CARGO_WORKSPACE_DIR`
+        // takes effect,
+
+        cargo.add_rustc_lib_path(builder);
+
+        // NOTE: we need to skip `src/tools/rust-analyzer/xtask` as they seem to exercise rustup /
+        // stable rustfmt.
+        //
+        // NOTE: you can only skip a specific workspace package via `--skip=...` if you *also*
+        // specify `--workspace`.
+        cargo.arg("--workspace");
+        cargo.arg("--exclude=xtask");
+
+        let mut skip_tests = vec![
+            // FIXME: may need a fix from https://github.com/rust-lang/rust-analyzer/pull/19124.
+            "config::tests::cargo_target_dir_subdir",
+        ];
+        skip_tests.extend(if self.host.starts_with("aarch64-apple") {
+            // FIXME: fails on `aarch64-apple` due to `cfg` differences
+            vec![
+                "item_tree::tests::generics_with_attributes",
+                "hover::tests::generic_params_disabled_by_cfg",
+            ]
+        } else if self.host == "i686-pc-windows-gnu" {
+            // FIXME: fails on `i686-mingw` due to multiple reasons. E.g. unsupport op on 32-bit
+            // windows-gnu target but on a 64-bit host, i.e. ptr width being different is
+            // unsupported.
+            vec![
+                "consteval::tests::add",
+                "consteval::tests::anonymous_const_block",
+                "consteval::tests::alignment",
+                "consteval::tests::bit_op",
+                "consteval::tests::associated_types",
+                "consteval::tests::boxes",
+                "consteval::tests::array_and_index",
+                "consteval::tests::builtin_derive_macro",
+                "consteval::tests::casts",
+                "consteval::tests::const_generic_subst_assoc_const_impl",
+                "consteval::tests::c_string",
+                "consteval::tests::const_generic_subst_fn",
+                "consteval::tests::const_impl_assoc",
+                "consteval::tests::closure_clone",
+                "consteval::tests::byte_string",
+                "consteval::tests::closures",
+                "consteval::tests::const_trait_assoc",
+                "consteval::tests::coerce_unsized",
+                "consteval::tests::consts",
+                "consteval::tests::enums",
+                "consteval::tests::floating_point",
+                "consteval::tests::destructing_assignment",
+                "consteval::tests::floating_point_casts",
+                "consteval::tests::closure_and_impl_fn",
+                "consteval::tests::extern_weak_statics",
+                "consteval::tests::function_call",
+                "consteval::tests::function_param_patterns",
+                "consteval::tests::closure_capture_unsized_type",
+                "consteval::tests::function_pointer",
+                "consteval::tests::from_ne_bytes",
+                "consteval::tests::function_pointer_in_constants",
+                "consteval::tests::from_trait",
+                "consteval::tests::ifs",
+                "consteval::tests::generic_fn",
+                "consteval::tests::dyn_trait",
+                "consteval::tests::impl_trait",
+                "consteval::tests::enum_variant_as_function",
+                "consteval::tests::const_transfer_memory",
+                "consteval::tests::intrinsics::allocator",
+                "consteval::tests::function_traits",
+                "consteval::tests::intrinsics::copy_nonoverlapping",
+                "consteval::tests::intrinsics::ctlz",
+                "consteval::tests::intrinsics::ctpop",
+                "consteval::tests::intrinsics::cttz",
+                "consteval::tests::intrinsics::const_eval_select",
+                "consteval::tests::intrinsics::floating_point",
+                "consteval::tests::for_loops",
+                "consteval::tests::function_pointer_and_niche_optimization",
+                "consteval::tests::intrinsics::likely",
+                "consteval::tests::intrinsics::arith_offset",
+                "consteval::tests::intrinsics::overflowing_add",
+                "consteval::tests::intrinsics::atomic",
+                "consteval::tests::intrinsics::rotate",
+                "consteval::tests::intrinsics::copy",
+                "consteval::tests::intrinsics::read_via_copy",
+                "consteval::tests::intrinsics::saturating",
+                "consteval::tests::intrinsics::needs_drop",
+                "consteval::tests::intrinsics::size_of",
+                "consteval::tests::intrinsics::simd",
+                "consteval::tests::intrinsics::min_align_of_val",
+                "consteval::tests::intrinsics::transmute",
+                "consteval::tests::intrinsics::type_name",
+                "consteval::tests::intrinsics::wrapping_add",
+                "consteval::tests::intrinsics::write_bytes",
+                "consteval::tests::intrinsics::write_via_move",
+                "consteval::tests::let_else",
+                "consteval::tests::layout_of_type_with_associated_type_field_defined_inside_body",
+                "consteval::tests::loops",
+                "consteval::tests::locals",
+                "consteval::tests::intrinsics::offset",
+                "consteval::tests::intrinsics::size_of_val",
+                "consteval::tests::or_pattern",
+                "consteval::tests::memory_limit",
+                "consteval::tests::intrinsics::ptr_offset_from",
+                "consteval::tests::intrinsics::discriminant_value",
+                "consteval::tests::path_pattern_matching",
+                "consteval::tests::overloaded_binop",
+                "consteval::tests::overloaded_deref",
+                "consteval::tests::overloaded_index",
+                "consteval::tests::overloaded_deref_autoref",
+                "consteval::tests::pattern_matching_ergonomics",
+                "consteval::tests::pattern_matching_literal",
+                "consteval::tests::pattern_matching_range",
+                "consteval::tests::manual_fn_trait_impl",
+                "consteval::tests::recursion",
+                "consteval::tests::reference_autoderef",
+                "consteval::tests::references",
+                "consteval::tests::ranges",
+                "consteval::tests::raw_pointer_equality",
+                "consteval::tests::match_guards",
+                "consteval::tests::structs",
+                "consteval::tests::options",
+                "consteval::tests::trait_method",
+                "consteval::tests::trait_basic",
+                "consteval::tests::pattern_matching_slice",
+                "consteval::tests::string",
+                "consteval::tests::trait_method_inside_block",
+                "consteval::tests::statics",
+                "consteval::tests::tuples",
+                "consteval::tests::unions",
+                "consteval::tests::result_layout_niche_optimization",
+                "consteval::tests::unsized_field",
+                "consteval::tests::try_block",
+                "consteval::tests::try_operator",
+                "mir::eval::tests::closure_capture_array_const_generic",
+                "mir::eval::tests::closure_layout_in_rpit",
+                "mir::eval::tests::drop_basic",
+                "mir::eval::tests::closure_state",
+                "mir::eval::tests::drop_in_place",
+                "mir::eval::tests::field_with_associated_type",
+                "mir::eval::tests::function_with_extern_c_abi",
+                "mir::eval::tests::generic_impl_for_trait_with_generic_method",
+                "mir::eval::tests::drop_if_let",
+                "mir::eval::tests::drop_struct_field",
+                "mir::eval::tests::index_of_slice_should_preserve_len",
+                "mir::eval::tests::for_loop",
+                "mir::eval::tests::manually_drop",
+                "mir::eval::tests::from_fn",
+                "mir::eval::tests::long_str_eq_same_prefix",
+                "mir::eval::tests::posix_getenv",
+                "mir::eval::tests::memcmp",
+                "mir::eval::tests::panic_display",
+                "mir::eval::tests::self_with_capital_s",
+                "mir::eval::tests::short_circuit_operator",
+                "mir::eval::tests::panic_fmt",
+                "mir::eval::tests::regression_14966",
+                "mir::eval::tests::posix_tls",
+                "mir::eval::tests::specialization_array_clone",
+                "mir::eval::tests::unix_write_stdout",
+                "mir::eval::tests::syscalls",
+                "tests::regression::regression_14456",
+                "tests::simple::const_eval_array_repeat_expr",
+                "tests::simple::const_eval_in_function_signature",
+                "tests::simple::issue_14275",
+                "tests::traits::array_length",
+                "tests::traits::const_generics",
+            ]
+        } else {
+            vec![]
+        });
+
+        let skip_tests =
+            skip_tests.into_iter().map(|name| format!("--skip={name}")).collect::<Vec<_>>();
+
+        run_cargo_test(
+            cargo,
+            &["--skip="],
+            &skip_tests,
+            "rust-analyzer",
+            "rust-analyzer",
+            self.host,
+            builder,
+        );
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct RustAnalyzerProcMacroSrv {
+    stage: u32,
+    host: TargetSelection,
+}
+
+impl Step for RustAnalyzerProcMacroSrv {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/rust-analyzer/crates/proc-macro-srv").alias("rust-analyzer")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target });
+    }
+
+    #[cfg_attr(
+        feature = "tracing",
+        instrument(
+            level = "debug",
+            name = "RustAnalyzerProcMacroSrv::run",
+            skip_all,
+            fields(stage = self.stage, host = ?self.host),
+        ),
+    )]
     fn run(self, builder: &Builder<'_>) {
         let stage = self.stage;
         let host = self.host;
+
+        debug!(stage, ?host, "ensuring compiler");
         let compiler = builder.compiler(stage, host);
 
         // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite,
         // but we do need the standard library to be present.
+        debug!(stage, ?compiler, "ensuring std");
         builder.ensure(compile::Rustc::new(compiler, host));
 
-        let workspace_path = "src/tools/rust-analyzer";
-        // until the whole RA test suite runs on `i686`, we only run
-        // `proc-macro-srv` tests
-        let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv";
+        debug!("running `cargo test` on `src/tools/rust-analyzer/crates/proc-macro-srv`");
+
         let mut cargo = tool::prepare_tool_cargo(
             builder,
             compiler,
             Mode::ToolRustc,
             host,
             Kind::Test,
-            crate_path,
+            "src/tools/rust-analyzer/crates/proc-macro-srv",
             SourceType::InTree,
             &["in-rust-tree".to_owned()],
         );
         cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES);
 
-        let dir = builder.src.join(workspace_path);
-        // needed by rust-analyzer to find its own text fixtures, cf.
-        // https://github.com/rust-analyzer/expect-test/issues/33
+        let dir = builder.src.join("src/tools/rust-analyzer");
+        // Needed by rust-analyzer to find its own text fixtures, cf.
+        // https://github.com/rust-analyzer/expect-test/issues/33.
         cargo.env("CARGO_WORKSPACE_DIR", &dir);
 
-        // RA's test suite tries to write to the source directory, that can't
-        // work in Rust CI
+        // RA's test suite tries to write to the source directory, that can't work in Rust CI.
         cargo.env("SKIP_SLOW_TESTS", "1");
 
         cargo.add_rustc_lib_path(builder);
-        run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", host, builder);
+        run_cargo_test(
+            cargo,
+            &[],
+            &[],
+            "rust-analyzer/crates/proc-macro-srv",
+            "rust-analyzer proc-macro-srv",
+            host,
+            builder,
+        );
     }
 }
 
@@ -2622,14 +2875,14 @@ fn prepare_cargo_test(
         cargo.arg("--quiet");
     }
 
-    // The tests are going to run with the *target* libraries, so we need to
-    // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent.
+    // The tests are going to run with the *target* libraries, so we need to ensure that those
+    // libraries show up in the LD_LIBRARY_PATH equivalent.
     //
-    // Note that to run the compiler we need to run with the *host* libraries,
-    // but our wrapper scripts arrange for that to be the case anyway.
+    // Note that to run the compiler we need to run with the *host* libraries, but our wrapper
+    // scripts arrange for that to be the case anyway.
     //
-    // We skip everything on Miri as then this overwrites the libdir set up
-    // by `Cargo::new` and that actually makes things go wrong.
+    // We skip everything on Miri as then this overwrites the libdir set up by `Cargo::new` and that
+    // actually makes things go wrong.
     if builder.kind != Kind::Miri {
         let mut dylib_path = dylib_path();
         dylib_path.insert(0, PathBuf::from(&*builder.sysroot_target_libdir(compiler, target)));

src/bootstrap/src/core/builder/mod.rs

@@ -21,7 +21,7 @@ use crate::core::config::{DryRun, TargetSelection};
 use crate::utils::cache::Cache;
 use crate::utils::exec::{BootstrapCommand, command};
 use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t};
-use crate::{Build, Crate};
+use crate::{Build, Crate, trace};
 
 mod cargo;
 
@@ -971,6 +971,7 @@ impl<'a> Builder<'a> {
                 test::Cargotest,
                 test::Cargo,
                 test::RustAnalyzer,
+                test::RustAnalyzerProcMacroSrv,
                 test::ErrorIndex,
                 test::Distcheck,
                 test::Nomicon,
@@ -1215,7 +1216,7 @@ impl<'a> Builder<'a> {
     /// `Compiler` since all `Compiler` instances are meant to be obtained through this function,
     /// since it ensures that they are valid (i.e., built and assembled).
     pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler {
-        self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } })
+        self.ensure(compile::Assemble { output_compiler: Compiler { stage, host } })
     }
 
     /// Similar to `compiler`, except handles the full-bootstrap option to
@@ -1331,6 +1332,8 @@ impl<'a> Builder<'a> {
             return;
         }
 
+        trace!(rustc_lib_paths = ?self.rustc_lib_paths(compiler));
+
         add_dylib_path(self.rustc_lib_paths(compiler), cmd);
     }