From 80b304d14beec80381bd11eb19099bb54d213092 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Fri, 27 Mar 2026 17:51:49 +0000 Subject: [PATCH] [DTLTO] Improve performance of adding files to the link (#186366) The in-process ThinLTO backend typically generates object files in memory and adds them directly to the link, except when the ThinLTO cache is in use. DTLTO is unusual in that it adds files to the link from disk in all cases. When the ThinLTO cache is not in use, ThinLTO adds files via an `AddStreamFn` callback provided by the linker, which ultimately appends to a `SmallVector` in LLD. When the cache is in use, the linker supplies an `AddBufferFn` callback that adds files more efficiently (by moving `MemoryBuffer` ownership). This patch adds a mandatory `AddBufferFn` to the DTLTO ThinLTO backend. The backend uses this to add files to the link more efficiently. Additionally: - Move AddStream from CGThinBackend to InProcessThinBackend, for reader clarity. - Modify linker comments that implied the AddBuffer path is cache-specific. For a Clang link (Debug build with sanitizers and instrumentation) using an optimized toolchain (PGO non-LTO, llvmorg-22.1.0), measuring the mean `Add DTLTO files to the link` time trace scope duration: - On Windows (Windows 11 Pro Build 26200, AMD Family 25 @ ~4.5 GHz, 16 cores/32 threads, 64 GB RAM), this patch reduces the mean from 2799.148 ms to 157.972 ms. - On Linux (Ubuntu 24.04.3 LTS Kernel 6.14, Ryzen 9 5950X, 16 cores/32 threads, boost up to 5.09 GHz, 64 GB RAM), this patch reduces the mean from 255.291 ms to 41.630 ms. Based on work by @romanova-ekaterina and @kbelochapka. --- lld/COFF/LTO.cpp | 26 ++++++++----- lld/ELF/LTO.cpp | 24 ++++++++---- llvm/include/llvm/LTO/LTO.h | 4 +- llvm/lib/LTO/LTO.cpp | 60 +++++++++++++++--------------- llvm/tools/llvm-lto2/llvm-lto2.cpp | 38 +++++++++---------- 5 files changed, 85 insertions(+), 67 deletions(-) diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index d55f95493a85..2d38a13830da 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -39,6 +39,16 @@ using namespace llvm::object; using namespace lld; using namespace lld::coff; +static AddBufferFn +createAddBufferFn(std::vector> &files, + std::vector &filenames) { + return [&files, &filenames](unsigned task, const Twine &moduleName, + std::unique_ptr mb) { + files[task] = std::move(mb); + filenames[task] = moduleName.str(); + }; +} + std::string BitcodeCompiler::getThinLTOOutputFile(StringRef path) { return lto::getThinLTOOutputFile(path, ctx.config.thinLTOPrefixReplaceOld, ctx.config.thinLTOPrefixReplaceNew); @@ -124,7 +134,8 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { /*ShouldEmitImportFiles=*/false, ctx.config.outputFile, ctx.config.dtltoDistributor, ctx.config.dtltoDistributorArgs, ctx.config.dtltoCompiler, ctx.config.dtltoCompilerPrependArgs, - ctx.config.dtltoCompilerArgs, !ctx.config.saveTempsArgs.empty()); + ctx.config.dtltoCompilerArgs, !ctx.config.saveTempsArgs.empty(), + createAddBufferFn(files, file_names)); } else if (ctx.config.thinLTOIndexOnly) { auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); }; backend = lto::createWriteIndexesThinBackend( @@ -200,11 +211,7 @@ std::vector BitcodeCompiler::compile() { FileCache cache; if (!ctx.config.ltoCache.empty()) cache = check(localCache("ThinLTO", "Thin", ctx.config.ltoCache, - [&](size_t task, const Twine &moduleName, - std::unique_ptr mb) { - files[task] = std::move(mb); - file_names[task] = moduleName.str(); - })); + createAddBufferFn(files, file_names))); checkError(ltoObj->run( [&](size_t task, const Twine &moduleName) { @@ -241,9 +248,10 @@ std::vector BitcodeCompiler::compile() { const char *Ext = emitASM ? ".s" : ".obj"; for (unsigned i = 0; i != maxTasks; ++i) { StringRef bitcodeFilePath; - // Get the native object contents either from the cache or from memory. Do - // not use the cached MemoryBuffer directly, or the PDB will not be - // deterministic. + // Get the native object contents either from a MemoryBuffer, for example + // from the cache or an external DTLTO backend compilation, or by reading + // from memory. Do not use the provided MemoryBuffer directly, or the PDB + // will not be deterministic. StringRef objBuf; if (files[i]) { objBuf = files[i]->getBuffer(); diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index ff8d44e0d7f6..352848af1551 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -38,6 +38,16 @@ using namespace llvm::ELF; using namespace lld; using namespace lld::elf; +static AddBufferFn +createAddBufferFn(std::vector> &files, + SmallVectorImpl &filenames) { + return [&files, &filenames](unsigned task, const Twine &moduleName, + std::unique_ptr mb) { + files[task] = std::move(mb); + filenames[task] = moduleName.str(); + }; +} + static std::string getThinLTOOutputFile(Ctx &ctx, StringRef modulePath) { return lto::getThinLTOOutputFile(modulePath, ctx.arg.thinLTOPrefixReplaceOld, ctx.arg.thinLTOPrefixReplaceNew); @@ -188,7 +198,7 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { ctx.arg.outputFile, ctx.arg.dtltoDistributor, ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler, ctx.arg.dtltoCompilerPrependArgs, ctx.arg.dtltoCompilerArgs, - !ctx.arg.saveTempsArgs.empty()); + !ctx.arg.saveTempsArgs.empty(), createAddBufferFn(files, filenames)); } else { backend = lto::createInProcessThinBackend( llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs), @@ -329,11 +339,7 @@ SmallVector, 0> BitcodeCompiler::compile() { FileCache cache; if (!ctx.arg.thinLTOCacheDir.empty()) cache = check(localCache("ThinLTO", "Thin", ctx.arg.thinLTOCacheDir, - [&](size_t task, const Twine &moduleName, - std::unique_ptr mb) { - files[task] = std::move(mb); - filenames[task] = moduleName.str(); - })); + createAddBufferFn(files, filenames))); if (!ctx.bitcodeFiles.empty()) checkError(ctx.e, ltoObj->run( @@ -386,8 +392,10 @@ SmallVector, 0> BitcodeCompiler::compile() { StringRef bitcodeFilePath; StringRef objBuf; if (files[i]) { - // When files[i] is not null, we get the native relocatable file from the - // cache. filenames[i] contains the original BitcodeFile's identifier. + // When files[i] is not null, it holds a native relocatable file provided + // as a MemoryBuffer, for example from the cache or from an external DTLTO + // backend compilation. filenames[i] contains the original BitcodeFile's + // identifier. objBuf = files[i]->getBuffer(); bitcodeFilePath = filenames[i]; } else { diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 38349b604b6b..3fec06b8bd78 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -367,13 +367,15 @@ LLVM_ABI ThinBackend createInProcessThinBackend( /// backend compilations. /// SaveTemps is a debugging tool that prevents temporary files created by this /// backend from being cleaned up. +/// AddBuffer is used to add a pre-existing native object buffer to the link. LLVM_ABI ThinBackend createOutOfProcessThinBackend( ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, StringRef LinkerOutputFile, StringRef Distributor, ArrayRef DistributorArgs, StringRef RemoteCompiler, ArrayRef RemoteCompilerPrependArgs, - ArrayRef RemoteCompilerArgs, bool SaveTemps); + ArrayRef RemoteCompilerArgs, bool SaveTemps, + AddBufferFn AddBuffer); /// This ThinBackend writes individual module indexes to files, instead of /// running the individual backend jobs. This backend is for distributed builds diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 36360081fa4a..1a6976eea808 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1540,7 +1540,6 @@ namespace { /// generated files back into the link. class CGThinBackend : public ThinBackendProc { protected: - AddStreamFn AddStream; DenseSet CfiFunctionDefs; DenseSet CfiFunctionDecls; bool ShouldEmitIndexFiles; @@ -1549,12 +1548,10 @@ public: CGThinBackend( const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, lto::IndexWriteCallback OnWrite, - bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, - ThreadPoolStrategy ThinLTOParallelism) + lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, + bool ShouldEmitImportsFiles, ThreadPoolStrategy ThinLTOParallelism) : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism), - AddStream(std::move(AddStream)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) { auto &Defs = CombinedIndex.cfiFunctionDefs(); CfiFunctionDefs.insert_range(Defs.guids()); @@ -1567,6 +1564,9 @@ public: /// an in-process thread when invoked for each task. class InProcessThinBackend : public CGThinBackend { protected: + // Callback used to add generated native object files to the link by code + // generating directly into the returned output stream. + AddStreamFn AddStream; FileCache Cache; public: @@ -1576,10 +1576,10 @@ public: const DenseMap &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles) - : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, - AddStream, OnWrite, ShouldEmitIndexFiles, - ShouldEmitImportsFiles, ThinLTOParallelism), - Cache(std::move(Cache)) {} + : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite, + ShouldEmitIndexFiles, ShouldEmitImportsFiles, + ThinLTOParallelism), + AddStream(std::move(AddStream)), Cache(std::move(Cache)) {} virtual Error runThinLTOBackendThread( AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM, @@ -2374,25 +2374,29 @@ class OutOfProcessThinBackend : public CGThinBackend { // Cache FileCache Cache; + // Callback to add a pre-existing native object buffer to the link. + AddBufferFn AddBuffer; + public: OutOfProcessThinBackend( const Config &Conf, ModuleSummaryIndex &CombinedIndex, ThreadPoolStrategy ThinLTOParallelism, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, FileCache CacheFn, lto::IndexWriteCallback OnWrite, + FileCache CacheFn, lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, StringRef LinkerOutputFile, StringRef Distributor, ArrayRef DistributorArgs, StringRef RemoteCompiler, ArrayRef RemoteCompilerPrependArgs, - ArrayRef RemoteCompilerArgs, bool SaveTemps) - : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, - AddStream, OnWrite, ShouldEmitIndexFiles, - ShouldEmitImportsFiles, ThinLTOParallelism), + ArrayRef RemoteCompilerArgs, bool SaveTemps, + AddBufferFn AddBuffer) + : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite, + ShouldEmitIndexFiles, ShouldEmitImportsFiles, + ThinLTOParallelism), LinkerOutputFile(LinkerOutputFile), DistributorPath(Distributor), DistributorArgs(DistributorArgs), RemoteCompiler(RemoteCompiler), RemoteCompilerPrependArgs(RemoteCompilerPrependArgs), RemoteCompilerArgs(RemoteCompilerArgs), SaveTemps(SaveTemps), - Cache(std::move(CacheFn)) {} + Cache(std::move(CacheFn)), AddBuffer(std::move(AddBuffer)) {} void setup(unsigned ThinLTONumTasks, unsigned ThinLTOTaskOffset, llvm::Triple Triple) override { @@ -2723,11 +2727,12 @@ public: Job.NativeObjectPath + ": " + EC.message(), inconvertibleErrorCode()); - MemoryBufferRef ObjFileMbRef = ObjFileMbOrErr->get()->getMemBufferRef(); if (Cache.isValid()) { // Cache hits are taken care of earlier. At this point, we could only // have cache misses. assert(Job.CacheAddStream); + MemoryBufferRef ObjFileMbRef = + ObjFileMbOrErr->get()->getMemBufferRef(); // Obtain a file stream for a storing a cache entry. auto CachedFileStreamOrErr = Job.CacheAddStream(Job.Task, Job.ModuleID); @@ -2743,13 +2748,7 @@ public: if (Error Err = CacheStream.commit()) return Err; } else { - auto StreamOrErr = AddStream(Job.Task, Job.ModuleID); - if (Error Err = StreamOrErr.takeError()) - report_fatal_error(std::move(Err)); - auto &Stream = *StreamOrErr->get(); - *Stream.OS << ObjFileMbRef.getBuffer(); - if (Error Err = Stream.commit()) - report_fatal_error(std::move(Err)); + AddBuffer(Job.Task, Job.ModuleID, std::move(*ObjFileMbOrErr)); } } } @@ -2764,17 +2763,18 @@ ThinBackend lto::createOutOfProcessThinBackend( StringRef LinkerOutputFile, StringRef Distributor, ArrayRef DistributorArgs, StringRef RemoteCompiler, ArrayRef RemoteCompilerPrependArgs, - ArrayRef RemoteCompilerArgs, bool SaveTemps) { + ArrayRef RemoteCompilerArgs, bool SaveTemps, + AddBufferFn AddBuffer) { auto Func = [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, FileCache Cache) { + AddStreamFn, FileCache Cache) { return std::make_unique( - Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, - AddStream, Cache, OnWrite, ShouldEmitIndexFiles, - ShouldEmitImportsFiles, LinkerOutputFile, Distributor, - DistributorArgs, RemoteCompiler, RemoteCompilerPrependArgs, - RemoteCompilerArgs, SaveTemps); + Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, Cache, + OnWrite, ShouldEmitIndexFiles, ShouldEmitImportsFiles, + LinkerOutputFile, Distributor, DistributorArgs, RemoteCompiler, + RemoteCompilerPrependArgs, RemoteCompilerArgs, SaveTemps, + AddBuffer); }; return ThinBackend(Func, Parallelism); } diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp index 95e8053dab32..8c676251cb39 100644 --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -411,6 +411,24 @@ static int run(int argc, char **argv) { auto DTLTOCompilerArgsSV = llvm::to_vector<0>(llvm::map_range( DTLTOCompilerArgs, [](const std::string &S) { return StringRef(S); })); + auto AddStream = + [&](size_t Task, + const Twine &ModuleName) -> std::unique_ptr { + std::string Path = OutputFilename + "." + utostr(Task); + + std::error_code EC; + auto S = std::make_unique(Path, EC, sys::fs::OF_None); + check(EC, Path); + return std::make_unique(std::move(S), Path); + }; + + auto AddBuffer = [&](size_t Task, const Twine &ModuleName, + std::unique_ptr MB) { + auto Stream = AddStream(Task, ModuleName); + *Stream->OS << MB->getBuffer(); + check(Stream->commit(), "Failed to commit cache"); + }; + ThinBackend Backend; if (ThinLTODistributedIndexes) Backend = createWriteIndexesThinBackend(llvm::hardware_concurrency(Threads), @@ -425,7 +443,7 @@ static int run(int argc, char **argv) { llvm::heavyweight_hardware_concurrency(Threads), /*OnWrite=*/{}, ThinLTOEmitIndexes, ThinLTOEmitImports, OutputFilename, DTLTODistributor, DTLTODistributorArgsSV, DTLTOCompiler, - DTLTOCompilerPrependArgsSV, DTLTOCompilerArgsSV, SaveTemps); + DTLTOCompilerPrependArgsSV, DTLTOCompilerArgsSV, SaveTemps, AddBuffer); } else Backend = createInProcessThinBackend( llvm::heavyweight_hardware_concurrency(Threads), @@ -496,24 +514,6 @@ static int run(int argc, char **argv) { if (HasErrors) return 1; - auto AddStream = - [&](size_t Task, - const Twine &ModuleName) -> std::unique_ptr { - std::string Path = OutputFilename + "." + utostr(Task); - - std::error_code EC; - auto S = std::make_unique(Path, EC, sys::fs::OF_None); - check(EC, Path); - return std::make_unique(std::move(S), Path); - }; - - auto AddBuffer = [&](size_t Task, const Twine &ModuleName, - std::unique_ptr MB) { - auto Stream = AddStream(Task, ModuleName); - *Stream->OS << MB->getBuffer(); - check(Stream->commit(), "Failed to commit cache"); - }; - FileCache Cache; if (!CacheDir.empty()) Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),