[DTLTO] Add DTLTO-specific LTO input handling time-trace scopes (#175799)

Add time-trace scopes to the DTLTO-specific input-handling code to
improve observability and debugging.

These scopes are tested via LLD, as the primary purpose of this code is
to support member files of non-thin archives as DTLTO inputs.
`llvm-lto2` does not currently support archives. Adding archive support
to `llvm-lto2` solely for testing these scopes does not appear to be
worthwhile.

As part of this change, the deletion of temporary DTLTO input files has
been moved. Cleanup now occurs after LTO has completed, rather than
during destruction of the LTO object. This is required since by the time
the LTO object is destroyed, time-traces have already been finalized, so
no additional trace data can be recorded.

Recording time-trace data for temporary file deletion is important, as
this has been a source of performance issues in the past and an area
where we expect to make further performance improvements if supported by
the data.

SIE internal tracker: TOOLCHAIN-21021

lld/test/ELF/dtlto/timetrace.test

@@ -0,0 +1,65 @@
+REQUIRES: x86
+
+## Test that DTLTO-specific LTO input file handling time-trace output is
+## produced as expected.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
+
+## Generate ThinLTO bitcode files.
+RUN: opt -thinlto-bc t1.ll -o t1.bc
+RUN: opt -thinlto-bc t2.ll -o t2.bc
+
+## Create archives.
+RUN: llvm-ar rcs t1.a t1.bc
+RUN: llvm-ar rcsT t2.thin.a t2.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o t1.o
+RUN: llc t2.ll --filetype=obj -o t2.o
+
+## Link and generate a time-trace.
+## Note: mock.py doesn't compile; it copies the specified object files to the
+## outputs in job order.
+RUN: ld.lld --whole-archive t1.a t2.thin.a -o my.elf \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=t1.o --thinlto-distributor-arg=t2.o \
+RUN:   --time-trace-granularity=0 --time-trace=%t.json
+RUN: %python filter_order_and_pprint.py %t.json | FileCheck %s
+
+## Check that DTLTO add input file events are recorded.
+CHECK:      "name": "Add input for DTLTO"
+CHECK:      "name": "Add input for DTLTO"
+CHECK:      "name": "Remove temporary inputs for DTLTO"
+CHECK:      "name": "Save input archive member for DTLTO"
+CHECK-SAME:   "detail": "t1.a(t1.bc at [[#ARCHIVE_OFFSET:]]).1.[[PID:[A-F0-9]+]].o"
+CHECK:      "name": "Total Add input for DTLTO"
+CHECK-SAME:   "count": 2,
+CHECK:      "name": "Total Remove temporary inputs for DTLTO"
+CHECK-SAME:   "count": 1,
+CHECK:      "name": "Total Save input archive member for DTLTO"
+CHECK-SAME:   "count": 1,
+
+#--- t1.ll
+target triple = "x86_64-unknown-linux-gnu"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @t1() {
+  ret void
+}
+
+#--- filter_order_and_pprint.py
+import json, sys
+
+data = json.load(open(sys.argv[1], "r", encoding="utf-8"))
+
+# Get DTLTO events.
+events = [e for e in data["traceEvents"] if "DTLTO" in e["name"]]
+events.sort(key=lambda e: (e["name"], str(e.get("args", {}).get("detail", ""))))
+
+# Print an event per line. Ensure 'name' is the first key.
+for ev in events:
+    name = ev.pop("name")
+    print(json.dumps({"name": name, **ev}))

llvm/include/llvm/DTLTO/DTLTO.h

@@ -16,19 +16,27 @@ namespace llvm {
 namespace lto {
 
 class DTLTO : public LTO {
+  using Base = LTO;
+
 public:
-  // Inherit contructors from LTO base class.
-  using LTO::LTO;
-  ~DTLTO() { removeTempFiles(); }
+  // Inherit constructors.
+  using Base::Base;
+  ~DTLTO() override = default;
+
+  // Add an input file and prepare it for distribution.
+  LLVM_ABI Expected<std::shared_ptr<InputFile>>
+  addInput(std::unique_ptr<InputFile> InputPtr) override;
+
+protected:
+  LLVM_ABI llvm::Error handleArchiveInputs() override;
+
+  LLVM_ABI void cleanup() override;
 
 private:
   // Bump allocator for a purpose of saving updated module IDs.
   BumpPtrAllocator PtrAlloc;
   StringSaver Saver{PtrAlloc};
 
-  // Removes temporary files.
-  LLVM_ABI void removeTempFiles();
-
   // Determines if a file at the given path is a thin archive file.
   Expected<bool> isThinArchive(const StringRef ArchivePath);
 
@@ -44,17 +52,8 @@ private:
 
   // A cache to avoid repeatedly reading the same archive file.
   StringMap<bool> ArchiveFiles;
-
-public:
-  // Adds the input file to the LTO object's list of input files.
-  // For archive members, generates a new module ID which is a path to a real
-  // file on a filesystem.
-  LLVM_ABI virtual Expected<std::shared_ptr<lto::InputFile>>
-  addInput(std::unique_ptr<lto::InputFile> InputPtr) override;
-
-  // Entry point for DTLTO archives support.
-  LLVM_ABI virtual llvm::Error handleArchiveInputs() override;
 };
+
 } // namespace lto
 } // namespace llvm
 

llvm/include/llvm/LTO/LTO.h

@@ -445,6 +445,13 @@ public:
   LLVM_ABI static SmallVector<const char *>
   getRuntimeLibcallSymbols(const Triple &TT);
 
+protected:
+  // Called at the start of run().
+  virtual Error handleArchiveInputs() { return Error::success(); }
+
+  // Called before returning from run().
+  virtual void cleanup() {}
+
 private:
   Config Conf;
 
@@ -622,8 +629,6 @@ public:
   addInput(std::unique_ptr<lto::InputFile> InputPtr) {
     return std::shared_ptr<lto::InputFile>(InputPtr.release());
   }
-
-  virtual llvm::Error handleArchiveInputs() { return llvm::Error::success(); }
 };
 
 /// The resolution for a symbol. The linker must provide a SymbolResolution for

llvm/lib/DTLTO/DTLTO.cpp

@@ -25,6 +25,7 @@
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
+#include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <iostream>
@@ -116,15 +117,6 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) {
   return IsThin;
 }
 
-// Removes any temporary regular archive member files that were created during
-// processing.
-void lto::DTLTO::removeTempFiles() {
-  for (auto &Input : InputFiles) {
-    if (Input->isMemberOfArchive())
-      sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);
-  }
-}
-
 // This function performs the following tasks:
 // 1. Adds the input file to the LTO object's list of input files.
 // 2. For thin archive members, generates a new module ID which is a path to a
@@ -133,6 +125,7 @@ void lto::DTLTO::removeTempFiles() {
 // 4. Updates the bitcode module's identifier.
 Expected<std::shared_ptr<lto::InputFile>>
 lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
+  TimeTraceScope TimeScope("Add input for DTLTO");
 
   // Add the input file to the LTO object.
   InputFiles.emplace_back(InputPtr.release());
@@ -180,6 +173,7 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
 Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) {
   StringRef ModuleId = Input->getName();
   if (Input->isMemberOfArchive()) {
+    TimeTraceScope TimeScope("Save input archive member for DTLTO", ModuleId);
     MemoryBufferRef MemoryBufferRef = Input->getFileBuffer();
     if (Error EC = saveBuffer(MemoryBufferRef.getBuffer(), ModuleId))
       return EC;
@@ -210,3 +204,14 @@ llvm::Error lto::DTLTO::handleArchiveInputs() {
     return EC;
   return Error::success();
 }
+
+// Remove temporary archive member files created to enable distribution.
+void lto::DTLTO::cleanup() {
+  {
+    TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
+    for (auto &Input : InputFiles)
+      if (Input->isMemberOfArchive())
+        sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);
+  }
+  Base::cleanup();
+}

llvm/lib/LTO/LTO.cpp

@@ -1217,6 +1217,8 @@ Error LTO::checkPartiallySplit() {
 }
 
 Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
+  llvm::scope_exit CleanUp([this]() { cleanup(); });
+
   if (Error EC = handleArchiveInputs())
     return EC;