Use a thread pool when building runtimes

toolchain/driver/BUILD

@@ -33,8 +33,11 @@ cc_library(
         "//toolchain/install:install_paths",
         "@llvm-project//clang:basic",
         "@llvm-project//clang:clang-driver",
+        "@llvm-project//clang:codegen",
         "@llvm-project//clang:driver",
         "@llvm-project//clang:frontend",
+        "@llvm-project//clang:frontend_tool",
+        "@llvm-project//clang:serialization",
         "@llvm-project//llvm:Core",
         "@llvm-project//llvm:Object",
         "@llvm-project//llvm:Support",

toolchain/driver/build_runtimes_subcommand.cpp

@@ -92,7 +92,8 @@ auto BuildRuntimesSubcommand::RunInternal(DriverEnv& driver_env)
                : Runtimes::Make(explicit_output_path, driver_env.vlog_stream));
   CARBON_ASSIGN_OR_RETURN(auto tmp_dir, Filesystem::MakeTmpDir());
 
-  return runner.BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path());
+  return runner.BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path(),
+                                       *driver_env.thread_pool);
 }
 
 }  // namespace Carbon

toolchain/driver/clang_runner.h

@@ -12,6 +12,7 @@
 #include "common/ostream.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/Triple.h"
 #include "toolchain/driver/runtimes_cache.h"
@@ -25,11 +26,15 @@ namespace Carbon {
 // incorporating custom command line flags from user invocations that we don't
 // parse, but will pass transparently along to Clang itself.
 //
+// This class is thread safe, allowing multiple threads to share a single runner
+// and concurrently invoke Clang.
+//
 // This doesn't literally use a subprocess to invoke Clang; it instead tries to
 // directly use the Clang command line driver library. We also work to simplify
 // how that driver operates and invoke it in an opinionated way to get the best
 // behavior for our expected use cases in the Carbon driver:
 //
+// - Ensure thread-safe invocation of Clang to enable concurrent usage.
 // - Minimize canonicalization of file names to try to preserve the paths as
 //   users type them.
 // - Minimize the use of subprocess invocations which are expensive on some
@@ -42,6 +47,13 @@ namespace Carbon {
 // standard output and standard error, and otherwise can only read and write
 // files based on their names described in the arguments. It doesn't provide any
 // higher-level abstraction such as streams for inputs or outputs.
+//
+// TODO: Switch the diagnostic machinery to buffer and do locked output so that
+// concurrent invocations of Clang don't intermingle their diagnostic output.
+//
+// TODO: If support for thread-local overrides of `llvm::errs` and `llvm::outs`
+// becomes available upstream, also buffer and synchronize those streams to
+// further improve the behavior of concurrent invocations.
 class ClangRunner : ToolRunnerBase {
  public:
   // Build a Clang runner that uses the provided `exe_name` and `err_stream`.
@@ -52,7 +64,8 @@ class ClangRunner : ToolRunnerBase {
               Runtimes::Cache* on_demand_runtimes_cache,
               llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs,
               llvm::raw_ostream* vlog_stream = nullptr,
-              bool build_runtimes_on_demand = false);
+              bool build_runtimes_on_demand = false,
+              llvm::ThreadPoolInterface* runtimes_build_thread_pool = nullptr);
 
   // Run Clang with the provided arguments.
   //
@@ -88,7 +101,8 @@ class ClangRunner : ToolRunnerBase {
   // return the path.
   auto BuildTargetResourceDir(const Runtimes::Cache::Features& features,
                               Runtimes& runtimes,
-                              const std::filesystem::path& tmp_path)
+                              const std::filesystem::path& tmp_path,
+                              llvm::ThreadPoolInterface& threads)
       -> ErrorOr<std::filesystem::path>;
 
   // Enable leaking memory.
@@ -103,6 +117,14 @@ class ClangRunner : ToolRunnerBase {
   auto EnableLeakingMemory() -> void { enable_leaking_ = true; }
 
  private:
+  // Emulates `cc1_main` but in a way that doesn't assume it is running in the
+  // main thread and can more easily fit into library calls to do compiles.
+  //
+  // TODO: Much of the logic here should be factored out of the CC1
+  // implementation in Clang's driver and into a reusable part of its libraries.
+  // That should allow reducing the code here to a minimal amount.
+  auto RunCC1(llvm::SmallVectorImpl<const char*>& cc1_args) -> int;
+
   // Handles building the Clang driver and passing the arguments down to it.
   auto RunInternal(llvm::ArrayRef<llvm::StringRef> args, llvm::StringRef target,
                    std::optional<llvm::StringRef> target_resource_dir_path)
@@ -125,15 +147,17 @@ class ClangRunner : ToolRunnerBase {
   auto BuildBuiltinsLib(llvm::StringRef target,
                         const llvm::Triple& target_triple,
                         const std::filesystem::path& tmp_path,
-                        Filesystem::DirRef lib_dir) -> ErrorOr<Success>;
+                        Filesystem::DirRef lib_dir,
+                        llvm::ThreadPoolInterface& threads) -> ErrorOr<Success>;
 
   Runtimes::Cache* runtimes_cache_;
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs_;
-  llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs> diagnostic_ids_;
 
   std::optional<std::filesystem::path> prebuilt_runtimes_path_;
 
+  llvm::ThreadPoolInterface* runtimes_build_thread_pool_ = nullptr;
+
   bool build_runtimes_on_demand_;
   bool enable_leaking_ = false;
 };

toolchain/driver/clang_runner_test.cpp

@@ -191,8 +191,9 @@ TEST_F(ClangRunnerTest, BuildResourceDir) {
   Runtimes::Cache::Features features = {.target = target};
   auto runtimes = *runtimes_cache_.Lookup(features);
   auto tmp_dir = *Filesystem::MakeTmpDir();
-  auto build_result =
-      runner.BuildTargetResourceDir(features, runtimes, tmp_dir.abs_path());
+  llvm::DefaultThreadPool threads(llvm::optimal_concurrency());
+  auto build_result = runner.BuildTargetResourceDir(
+      features, runtimes, tmp_dir.abs_path(), threads);
   ASSERT_TRUE(build_result.ok()) << build_result.error();
   std::filesystem::path resource_dir_path = std::move(*build_result);
 

toolchain/driver/clang_subcommand.cpp

@@ -72,7 +72,8 @@ auto ClangSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
   ClangRunner runner(
       driver_env.installation, &driver_env.runtimes_cache, driver_env.fs,
       driver_env.vlog_stream,
-      /*build_runtimes_on_demand=*/options_.build_runtimes_on_demand);
+      /*build_runtimes_on_demand=*/options_.build_runtimes_on_demand,
+      driver_env.thread_pool);
 
   // Don't run Clang when fuzzing, it is known to not be reliable under fuzzing
   // due to many unfixed issues.

toolchain/driver/driver.cpp

@@ -32,6 +32,7 @@ struct Options {
   bool verbose = false;
   bool fuzzing = false;
   bool include_diagnostic_kind = false;
+  bool threads = true;
 
   llvm::StringRef runtimes_cache_path;
   llvm::StringRef prebuilt_runtimes_path;
@@ -124,6 +125,25 @@ applies to each message that forms a diagnostic, not just the primary message.
       },
       [&](auto& arg_b) { arg_b.Set(&include_diagnostic_kind); });
 
+  b.AddFlag(
+      {
+          .name = "threads",
+          .help = R"""(
+Controls whether threads are used to build runtimes.
+
+When enabled (the default), Carbon will try to build runtime libraries using
+threads to parallelize the operation. How many threads is controlled
+automatically by the system.
+
+Disabling threads ensures a single threaded build of the runtimes which can help
+when there are errors or other output.
+)""",
+      },
+      [&](auto& arg_b) {
+        arg_b.Default(true);
+        arg_b.Set(&threads);
+      });
+
   runtimes.AddTo(b, &selected_subcommand);
   clang.AddTo(b, &selected_subcommand);
   compile.AddTo(b, &selected_subcommand);
@@ -208,6 +228,14 @@ auto Driver::RunCommand(llvm::ArrayRef<llvm::StringRef> args) -> DriverResult {
     driver_env_.fuzzing = true;
   }
 
+  llvm::SingleThreadExecutor single_thread({.ThreadsRequested = 1});
+  std::optional<llvm::DefaultThreadPool> threads;
+  driver_env_.thread_pool = &single_thread;
+  if (options.threads) {
+    threads.emplace(llvm::optimal_concurrency());
+    driver_env_.thread_pool = &*threads;
+  }
+
   CARBON_CHECK(options.selected_subcommand != nullptr);
   return options.selected_subcommand->Run(driver_env_);
 }

toolchain/driver/driver_env.h

@@ -9,6 +9,8 @@
 #include <utility>
 
 #include "common/ostream.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/Threading.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "toolchain/diagnostics/diagnostic_emitter.h"
 #include "toolchain/driver/runtimes_cache.h"
@@ -62,6 +64,9 @@ struct DriverEnv {
   // A diagnostic emitter that has no locations.
   Diagnostics::NoLocEmitter emitter;
 
+  // Thread pool available for use when concurrency is needed.
+  llvm::ThreadPoolInterface* thread_pool;
+
   // For CARBON_VLOG.
   llvm::raw_pwrite_stream* vlog_stream = nullptr;