[DTLTO] Speed up temporary file removal in the ThinLTO backend (#186988)

Deleting the temporary files produced by the DTLTO ThinLTO backend can
be expensive on Windows hosts. For a Clang link (Debug build with
sanitizers and instrumentation) using an optimized toolchain (PGO
non-LTO, llvmorg-22.1.0) on a Windows 11 Pro (Build 26200), AMD Family
25 @ ~4.5 GHz, 16 cores / 32 threads, 64 GB RAM machine, the mean
duration of the "Remove DTLTO temporary files" time trace scope was
1267.789 ms (measured over 10 runs).

This patch performs the deletions on a background thread, allowing them
to overlap with the remainder of the link to hide this cost.

Based on work by @romanova-ekaterina and @kbelochapka.

llvm/lib/LTO/LTO.cpp

@@ -43,6 +43,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/JSON.h"
+#include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
@@ -2321,6 +2322,45 @@ std::vector<int> lto::generateModulesOrdering(ArrayRef<BitcodeModule *> R) {
 }
 
 namespace {
+// There can be many temporary files to remove. Performing deletion in
+// the background can save a few seconds on Windows hosts. Experimentation
+// showed that serial deletion is most efficient, hence a single thread.
+struct BackgroundDeletion : llvm::DefaultThreadPool {
+  BackgroundDeletion()
+      : llvm::DefaultThreadPool(llvm::heavyweight_hardware_concurrency(1)) {}
+
+  ~BackgroundDeletion() {
+    wait();
+    for (const std::string &Warning : Warnings)
+      errs() << "warning: could not remove the file " << Warning << "\n";
+  }
+
+  void remove(SmallVector<std::string> &&Files, const Config &Conf) {
+    async([this, Files = std::move(Files), TTE = Conf.TimeTraceEnabled,
+           TTG = Conf.TimeTraceGranularity] {
+      if (LLVM_ENABLE_THREADS && TTE)
+        timeTraceProfilerInitialize(TTG, "Remove DTLTO temporary files");
+      {
+        llvm::TimeTraceScope TimeScope("Remove DTLTO temporary files");
+        for (const auto &F : Files) {
+          std::error_code EC = sys::fs::remove(F, true);
+          if (EC &&
+              EC != std::make_error_code(std::errc::no_such_file_or_directory))
+            Warnings.emplace_back("'" + F + "': " + EC.message());
+        }
+      }
+      if (LLVM_ENABLE_THREADS && TTE)
+        timeTraceProfilerFinishThread();
+    });
+  }
+
+  SmallVector<std::string> Warnings;
+};
+
+// Use a ManagedStatic to allow the thread to run until llvm_shutdown() is
+// called for the current process.
+static llvm::ManagedStatic<BackgroundDeletion> BackgroundDeleter;
+
 /// This out-of-process backend does not perform code generation when invoked
 /// for each task. Instead, it generates the necessary information (e.g., the
 /// summary index shard, import list, etc.) to enable code generation to be
@@ -2653,13 +2693,15 @@ public:
       return std::move(*Err);
 
     llvm::scope_exit CleanPerJobFiles([&] {
-      llvm::TimeTraceScope TimeScope("Remove DTLTO temporary files");
-      if (!SaveTemps)
-        for (auto &Job : Jobs) {
-          removeFile(Job.NativeObjectPath);
-          if (!ShouldEmitIndexFiles)
-            removeFile(Job.SummaryIndexPath);
-        }
+      if (SaveTemps)
+        return;
+      SmallVector<std::string> Files;
+      for (auto &Job : Jobs) {
+        Files.push_back(std::string(Job.NativeObjectPath));
+        if (!ShouldEmitIndexFiles)
+          Files.push_back(std::string(Job.SummaryIndexPath));
+      }
+      BackgroundDeleter->remove(std::move(Files), Conf);
     });
 
     const StringRef BCError = "DTLTO backend compilation: ";

llvm/tools/llvm-lto2/llvm-lto2.cpp

@@ -26,6 +26,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/PluginLoader.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/Threading.h"
@@ -284,7 +285,11 @@ static int run(int argc, char **argv) {
 
   if (TimeTrace)
     timeTraceProfilerInitialize(TimeTraceGranularity, argv[0]);
-  llvm::scope_exit TimeTraceScopeExit([]() {
+  llvm::scope_exit ShutdownScopeExit([]() {
+    // llvm_shutdown must be called before finalizing the time trace to
+    // ensure that time trace scopes from ManagedStatic destructors are
+    // flushed and recorded.
+    llvm::llvm_shutdown();
     if (TimeTrace) {
       check(timeTraceProfilerWrite(TimeTraceFile, OutputFilename),
             "timeTraceProfilerWrite failed");