[DTLTO][LLD][ELF] Add support for Integrated Distributed ThinLTO (#142757)

This patch introduces support for Integrated Distributed ThinLTO (DTLTO) in ELF LLD. DTLTO enables the distribution of ThinLTO backend compilations via external distribution systems, such as Incredibuild, during the traditional link step: https://llvm.org/docs/DTLTO.html. It is expected that users will invoke DTLTO through the compiler driver (e.g., Clang) rather than calling LLD directly. A Clang-side interface for DTLTO will be added in a follow-up patch. Note: Bitcode members of archives (thin or non-thin) are not currently supported. This will be addressed in a future change. As a consequence of this lack of support, this patch is not sufficient to allow for self-hosting an LLVM build with DTLTO. Theoretically, --start-lib/--end-lib could be used instead of archives in a self-host build. However, it's unclear how --start-lib/--end-lib can be easily used with the LLVM build system. Testing: - ELF LLD `lit` test coverage has been added, using a mock distributor to avoid requiring Clang. - Cross-project `lit` tests cover integration with Clang. For the design discussion of the DTLTO feature, see: #126654.
2025-07-02 16:12:27 +01:00 · 2025-07-02 16:12:27 +01:00 · 3b4e79398d
commit 3b4e79398d
parent e32439249d
14 changed files with 306 additions and 1 deletions
--- a/cross-project-tests/CMakeLists.txt
+++ b/cross-project-tests/CMakeLists.txt
@ -94,6 +94,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
  DEPENDS clang
  )

+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
--- a/cross-project-tests/dtlto/README.md
+++ b/cross-project-tests/dtlto/README.md
@ -0,0 +1,3 @@
+Tests for DTLTO (Integrated Distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.
--- a/cross-project-tests/dtlto/ld-dtlto.c
+++ b/cross-project-tests/dtlto/ld-dtlto.c
@ -0,0 +1,41 @@
+// REQUIRES: ld.lld
+
+/// Simple test that DTLTO works with a single input bitcode file and that
+/// --save-temps can be applied to the remote compilation.
+
+// RUN: rm -rf %t && mkdir %t && cd %t
+
+// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s -o dtlto.o
+
+// RUN: ld.lld dtlto.o \
+// RUN:   --thinlto-distributor=%python \
+// RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+// RUN:   --thinlto-remote-compiler=%clang \
+// RUN:   --thinlto-remote-compiler-arg=--save-temps
+
+/// Check that the required output files have been created.
+// RUN: ls | sort | FileCheck %s
+
+/// No files are expected before.
+// CHECK-NOT: {{.}}
+
+/// Linked ELF.
+// CHECK: {{^}}a.out{{$}}
+
+/// Produced by the bitcode compilation.
+// CHECK-NEXT: {{^}}dtlto.o{{$}}
+
+/// --save-temps output for the backend compilation.
+// CHECK-NEXT: {{^}}dtlto.s{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}}
+
+/// No files are expected after.
+// CHECK-NOT: {{.}}
+
+int _start() { return 0; }
--- a/cross-project-tests/dtlto/lit.local.cfg
+++ b/cross-project-tests/dtlto/lit.local.cfg
@ -0,0 +1,5 @@
+if any(
+    f not in config.available_features
+    for f in ("clang", "x86-registered-target")
+):
+    config.unsupported = True
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@ -275,6 +275,10 @@ struct Config {
  llvm::SmallVector<llvm::StringRef, 0> searchPaths;
  llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
  llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
+  llvm::StringRef dtltoDistributor;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
+  llvm::StringRef dtltoCompiler;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
  llvm::SmallVector<llvm::StringRef, 0> undefined;
  llvm::SmallVector<SymbolVersion, 0> dynamicList;
  llvm::SmallVector<uint8_t, 0> buildIdVector;
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@ -1396,6 +1396,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
      args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
  ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
  ctx.arg.discard = getDiscard(args);
+  ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq);
+  ctx.arg.dtltoDistributorArgs =
+      args::getStrings(args, OPT_thinlto_distributor_arg);
+  ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq);
+  ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_arg);
  ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
  ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
  ctx.arg.ehFrameHdr =
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@ -180,6 +180,13 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
        std::string(ctx.arg.thinLTOPrefixReplaceNew),
        std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
        ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
+  } else if (!ctx.arg.dtltoDistributor.empty()) {
+    backend = lto::createOutOfProcessThinBackend(
+        llvm::hardware_concurrency(ctx.arg.thinLTOJobs), onIndexWrite,
+        ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles,
+        ctx.arg.outputFile, ctx.arg.dtltoDistributor,
+        ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler,
+        ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty());
  } else {
    backend = lto::createInProcessThinBackend(
        llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@ -714,7 +714,17 @@ def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
 def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
  HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">;
-
+def thinlto_distributor_eq: JJ<"thinlto-distributor=">,
+  HelpText<"Distributor to use for ThinLTO backend compilations. If specified, "
+  "ThinLTO backend compilations will be distributed">;
+defm thinlto_distributor_arg: EEq<"thinlto-distributor-arg", "Arguments to "
+  "pass to the ThinLTO distributor">;
+def thinlto_compiler_eq: JJ<"thinlto-remote-compiler=">,
+  HelpText<"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
+  "compilations">;
+defm thinlto_compiler_arg: EEq<"thinlto-remote-compiler-arg", "Compiler "
+  "arguments for the ThinLTO distributor to pass for ThinLTO backend "
+  "compilations">;
 defm fat_lto_objects: BB<"fat-lto-objects",
    "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.",
    "Ignore the .llvm.lto section in relocatable object files (default).">;
--- a/lld/docs/DTLTO.rst
+++ b/lld/docs/DTLTO.rst
@ -0,0 +1,42 @@
+Integrated Distributed ThinLTO (DTLTO)
+======================================
+
+Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend
+ThinLTO compilations via external distribution systems, such as Incredibuild,
+during the traditional link step.
+
+The implementation is documented here: https://llvm.org/docs/DTLTO.html.
+
+Currently, DTLTO is only supported in ELF LLD. Support will be added to other
+LLD flavours in the future.
+
+ELF LLD
+-------
+
+The command-line interface is as follows:
+
+- ``--thinlto-distributor=<path>``  
+  Specifies the file to execute as the distributor process. If specified,
+  ThinLTO backend compilations will be distributed.
+
+- ``--thinlto-remote-compiler=<path>``  
+  Specifies the path to the compiler that the distributor process will use for
+  backend compilations. The compiler invoked must match the version of LLD.
+
+- ``--thinlto-distributor-arg=<arg>``  
+  Specifies ``<arg>`` on the command line when invoking the distributor.
+  Can be specified multiple times.
+
+- ``--thinlto-remote-compiler-arg=<arg>``  
+  Appends ``<arg>`` to the remote compiler's command line.
+  Can be specified multiple times.
+
+  Options that introduce extra input/output files may cause miscompilation if
+  the distribution system does not automatically handle pushing/fetching them to
+  remote nodes. In such cases, configure the distributor - possibly using
+  ``--thinlto-distributor-arg=`` - to manage these dependencies. See the
+  distributor documentation for details.
+
+Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported.
+Currently, other options are silently accepted but do not have the intended
+effect. Support for such options will be expanded in the future.
--- a/lld/docs/index.rst
+++ b/lld/docs/index.rst
@ -147,3 +147,4 @@ document soon.
   ELF/start-stop-gc
   ELF/warn_backrefs
   MachO/index
+   DTLTO
--- a/lld/test/ELF/dtlto/files.test
+++ b/lld/test/ELF/dtlto/files.test
@ -0,0 +1,99 @@
+# REQUIRES: x86
+
+## Test that the LLD options --save-temps, --thinlto-emit-index-files,
+## and --thinlto-emit-imports-files function correctly with DTLTO.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
+
+## Generate ThinLTO bitcode files. Note that t3.bc will not be used by the
+## linker.
+RUN: opt -thinlto-bc t1.ll -o t1.bc
+RUN: opt -thinlto-bc t2.ll -o t2.bc
+RUN: cp t1.bc t3.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o t1.o
+RUN: llc t2.ll --filetype=obj -o t2.o
+
+## Create response file containing shared ThinLTO linker arguments.
+## --start-lib/--end-lib is used to test the special case where unused lazy
+## bitcode inputs result in empty index/imports files.
+## Note that mock.py does not do any compilation; instead, it simply writes
+## the contents of the object files supplied on the command line into the
+## output object files in job order.
+RUN: echo "t1.bc t2.bc --start-lib t3.bc --end-lib -o my.elf \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=t1.o \
+RUN:   --thinlto-distributor-arg=t2.o" > l.rsp
+
+## Check that without extra flags, no index/imports files are produced and
+## backend temp files are removed.
+RUN: ld.lld @l.rsp
+RUN: ls | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,NOINDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that index files are created with --thinlto-emit-index-files.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-index-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,INDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that imports files are created with --thinlto-emit-imports-files.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,NOINDEXFILES,IMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that both index and imports files are emitted with both flags.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-index-files \
+RUN:   --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,INDEXFILES,IMPORTSFILES,EMPTYIMPORTS
+
+## Check that backend temp files are retained with --save-temps.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --save-temps
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=BACKEND,NOINDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that all files are emitted when all options are enabled.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --save-temps --thinlto-emit-index-files \
+RUN:   --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=BACKEND,INDEXFILES,IMPORTSFILES,EMPTYIMPORTS
+
+## JSON jobs description, retained with --save-temps.
+## Note that DTLTO temporary files include a PID component.
+NOBACKEND-NOT:      {{^}}my.[[#]].dist-file.json{{$}}
+BACKEND:            {{^}}my.[[#]].dist-file.json{{$}}
+
+## Index/imports files for t1.bc.
+NOIMPORTSFILES-NOT: {{^}}t1.bc.imports{{$}}
+IMPORTSFILES:       {{^}}t1.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t1.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t1.bc.thinlto.bc{{$}}
+
+## Index/imports files for t2.bc.
+NOIMPORTSFILES-NOT: {{^}}t2.bc.imports{{$}}
+IMPORTSFILES:       {{^}}t2.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t2.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t2.bc.thinlto.bc{{$}}
+
+## Empty index/imports files for unused t3.bc.
+NOEMPTYIMPORTS-NOT: {{^}}t3.bc.imports{{$}}
+EMPTYIMPORTS:       {{^}}t3.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t3.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t3.bc.thinlto.bc{{$}}
+
+#--- t1.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t1() {
+  ret void
+}
--- a/lld/test/ELF/dtlto/options.test
+++ b/lld/test/ELF/dtlto/options.test
@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## Test that DTLTO options are passed correctly to the distributor and
+## remote compiler.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: opt -thinlto-bc foo.ll -o foo.o
+
+## Note: validate.py does not perform any compilation. Instead, it validates the
+## received JSON, pretty-prints the JSON and the supplied arguments, and then
+## exits with an error. This allows FileCheck directives to verify the
+## distributor inputs.
+RUN: not ld.lld foo.o \
+RUN:   -o my.elf \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
+RUN:   --thinlto-distributor-arg=darg1=10 \
+RUN:   --thinlto-distributor-arg=darg2=20 \
+RUN:   --thinlto-remote-compiler=my_clang.exe \
+RUN:   --thinlto-remote-compiler-arg=carg1=20 \
+RUN:   --thinlto-remote-compiler-arg=carg2=30 2>&1 | FileCheck %s
+
+CHECK: distributor_args=['darg1=10', 'darg2=20']
+
+CHECK: "linker_output": "my.elf"
+
+CHECK: "my_clang.exe"
+CHECK: "carg1=20"
+CHECK: "carg2=30"
+
+CHECK: error: DTLTO backend compilation: cannot open native object file:
+
+#--- foo.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo() {
+  ret void
+}
--- a/lld/test/ELF/dtlto/partitions.test
+++ b/lld/test/ELF/dtlto/partitions.test
@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## Test that DTLTO works with more than one LTO partition.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@f/@t1/g' f.ll > t1.ll
+RUN: sed 's/@f/@t2/g' f.ll > t2.ll
+
+## Generate bitcode.
+RUN: opt f.ll -o full.bc
+RUN: opt -thinlto-bc t1.ll -o thin1.bc
+RUN: opt -thinlto-bc t2.ll -o thin2.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o thin1.o
+RUN: llc t2.ll --filetype=obj -o thin2.o
+
+## Link with 3 LTO partitions.
+RUN: ld.lld full.bc thin1.bc thin2.bc \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=thin1.o \
+RUN:   --thinlto-distributor-arg=thin2.o \
+RUN:   --save-temps \
+RUN:   --lto-partitions=3
+
+## DTLTO temporary object files include the task number and a PID component. The
+## task number should incorporate the LTO partition number.
+RUN: ls | sort | FileCheck %s
+CHECK: {{^}}thin1.3.[[PID:[a-zA-Z0-9_]+]].native.o{{$}}
+CHECK: {{^}}thin2.4.[[PID]].native.o{{$}}
+
+#--- f.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() {
+  ret void
+}
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@ -36,6 +36,7 @@ config.test_exec_root = os.path.join(config.lld_obj_root, "test")

 llvm_config.use_default_substitutions()
 llvm_config.use_lld()
+config.substitutions.append(("%llvm_src_root", config.llvm_src_root))

 tool_patterns = [
    "llc",