shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[ThinLTO] a ThinLTO warning is added if cache_size_bytes or cache_size_files is too small for the current link job. The warning recommends the user to consider adjusting --thinlto-cache-policy.

Browse Source 
A specific case for ThinLTO cache pruning is that the current build is huge, and the cache wasn't big enough to hold the intermediate object files of that build. So in doing that build, a file would be cached, and later in that same build it would be evicted. This was significantly decreasing the effectiveness of the cache. By giving this warning, the user could identify the required cache size/files and improve ThinLTO link speed.

Differential Revision: https://reviews.llvm.org/D135590
This commit is contained in:
Ying Yi 2022-10-10 13:26:56 +01:00
parent eb05da5eed
commit ca875cb4fe
11 changed files with 230 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lld/COFF/LTO.cpp
View File

@ -202,7 +202,7 @@ std::vector<InputFile *> BitcodeCompiler::compile(COFFLinkerContext &ctx) {
}
if (!config->ltoCache.empty())
pruneCache(config->ltoCache, config->ltoCachePolicy);
pruneCache(config->ltoCache, config->ltoCachePolicy, files);
std::vector<InputFile *> ret;
for (unsigned i = 0; i != maxTasks; ++i) {

2
lld/ELF/LTO.cpp
View File

@ -367,7 +367,7 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
}
if (!config->thinLTOCacheDir.empty())
pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy);
pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy, files);
if (!config->ltoObjPath.empty()) {
saveBuffer(buf[0], config->ltoObjPath);

2
lld/MachO/LTO.cpp
View File

@ -141,7 +141,7 @@ std::vector<ObjFile *> BitcodeCompiler::compile() {
cache));
if (!config->thinLTOCacheDir.empty())
pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy);
pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy, files);
// In ThinLTO mode, Clang passes a temporary directory in -object_path_lto,
// while the argument is a single file in FullLTO mode.

43
lld/test/COFF/lto-cache-warnings.ll Normal file
View File

@ -0,0 +1,43 @@
; REQUIRES: x86, shell
; RUN: opt -module-hash -module-summary %s -o %t.o
; RUN: opt -module-hash -module-summary %p/Inputs/lto-cache.ll -o %t2.o
; RUN: rm -rf %t && mkdir %t
;; Check cache policies of the number of files.
;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning.
; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_files=0 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck --implicit-check-not=warning: %s
;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning.
; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_files=3 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given.
; RUN: lld-link /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_files=1 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --check-prefix=NUM
;; Check cache policies of the cache size.
;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning.
; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_bytes=0 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Get the total size of created cache files.
; RUN: rm -rf %t && mkdir %t && cd %t
; RUN: lld-link /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_bytes=32k /out:%t3 /entry:main %t2.o %t.o 2>&1
; RUN: %python -c "import os, sys; print(sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')))" > %t.size.txt
;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning.
; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) + 5)) /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given.
; RUN: lld-link /verbose /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) - 5)) /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --check-prefix=SIZE
;; Check emit two warnings if pruning happens due to reach both the size and number limits.
; RUN: lld-link /lldltocache:%t /lldltocachepolicy:prune_interval=0s:cache_size_files=1:cache_size_bytes=1 /out:%t3 /entry:main %t2.o %t.o 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE
; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy
; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy
target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-pc-windows-msvc"
define void @globalfunc() #0 {
entry:
ret void
}

44
lld/test/ELF/lto/cache-warnings.ll Normal file
View File

@ -0,0 +1,44 @@
; REQUIRES: x86, shell
; RUN: opt -module-hash -module-summary %s -o %t.o
; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o
; RUN: rm -rf %t && mkdir %t
;; Check cache policies of the number of files.
;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning.
; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=0 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning.
; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=3 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given.
; RUN: ld.lld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,WARN
;; Check cache policies of the cache size.
;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning.
; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=0 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Get the total size of created cache files.
; RUN: rm -rf %t && mkdir %t && cd %t
; RUN: ld.lld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=32k %t2.o %t.o -o %t3 2>&1
; RUN: %python -c "import os, sys; print(sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')))" > %t.size.txt
;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning.
; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) + 5)) %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given.
; RUN: ld.lld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) - 5)) %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=SIZE,WARN
;; Check emit two warnings if pruning happens due to reach both the size and number limits.
; RUN: ld.lld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1:cache_size_bytes=1 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE,WARN
; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy
; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy
; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
define void @globalfunc() {
entry:
ret void
}

59
lld/test/MachO/lto-cache-warnings.ll Normal file
View File

@ -0,0 +1,59 @@
; REQUIRES: x86, shell
; RUN: rm -rf %t; split-file %s %t
; RUN: opt -module-hash -module-summary %t/foo.ll -o %t/foo.o
; RUN: opt -module-hash -module-summary %t/bar.ll -o %t/bar.o
;; Check cache policies of the number of files.
;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning.
; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_files=0 -o %t/test %t/foo.o %t/bar.o 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning.
; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_files=3 %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given.
; RUN: %lld -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1 %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --check-prefixes=NUM,WARN
;; Check cache policies of the cache size.
;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning.
; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=0 %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Get the total size of created cache files.
; RUN: cd %t
; RUN: %lld -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=32k %t/foo.o %t/bar.o -o %t/test 2>&1
; RUN: %python -c "import os, sys; print(sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')))" > %t.size.txt
;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning.
; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) + 5)) %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given.
; RUN: %lld -v -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) - 5)) %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --check-prefixes=SIZE,WARN
;; Check emit two warnings if pruning happens due to reach both the size and number limits.
; RUN: %lld -cache_path_lto %t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1:cache_size_bytes=1 %t/foo.o %t/bar.o -o %t/test 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE,WARN
; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy
; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy
; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy
;--- foo.ll
target triple = "x86_64-apple-macosx10.15.0"
target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
define void @globalfunc() #0 {
entry:
ret void
}
;--- bar.ll
target triple = "x86_64-apple-macosx10.15.0"
target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
define i32 @main() {
entry:
call void (...) @globalfunc()
ret i32 0
}
declare void @globalfunc(...)

44
lld/test/wasm/lto/cache-warnings.ll Normal file
View File

@ -0,0 +1,44 @@
; REQUIRES: shell
; RUN: opt -module-hash -module-summary %s -o %t.o
; RUN: opt -module-hash -module-summary %p/Inputs/cache.ll -o %t2.o
; RUN: rm -rf %t && mkdir %t
;; Check cache policies of the number of files.
;; Case 1: A value of 0 disables the number of files based pruning. Therefore, there is no warning.
; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=0 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 2: If the total number of the files created by the current link job is less than the maximum number of files, there is no warning.
; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=3 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 3: If the total number of the files created by the current link job exceeds the maximum number of files, a warning is given.
; RUN: wasm-ld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,WARN
;; Check cache policies of the cache size.
;; Case 1: A value of 0 disables the absolute size-based pruning. Therefore, there is no warning.
; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=0 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Get the total size of created cache files.
; RUN: rm -rf %t && mkdir %t && cd %t
; RUN: wasm-ld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=32k %t2.o %t.o -o %t3 2>&1
; RUN: %python -c "import os, sys; print(sum(os.path.getsize(filename) for filename in os.listdir('.') if os.path.isfile(filename) and filename.startswith('llvmcache-')))" > %t.size.txt
;; Case 2: If the total size of the cache files created by the current link job is less than the maximum size for the cache directory in bytes, there is no warning.
; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) + 5)) %t2.o %t.o -o %t3 2>&1 | FileCheck %s --implicit-check-not=warning:
;; Case 3: If the total size of the cache files created by the current link job exceeds the maximum size for the cache directory in bytes, a warning is given.
; RUN: wasm-ld --verbose --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_bytes=$(($(cat %t.size.txt) - 5)) %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=SIZE,WARN
;; Check emit two warnings if pruning happens due to reach both the size and number limits.
; RUN: wasm-ld --thinlto-cache-dir=%t --thinlto-cache-policy=prune_interval=0s:cache_size_files=1:cache_size_bytes=1 %t2.o %t.o -o %t3 2>&1 | FileCheck %s --check-prefixes=NUM,SIZE,WARN
; NUM: warning: ThinLTO cache pruning happens since the number of{{.*}}--thinlto-cache-policy
; SIZE: warning: ThinLTO cache pruning happens since the total size of{{.*}}--thinlto-cache-policy
; WARN-NOT: warning: ThinLTO cache pruning happens{{.*}}--thinlto-cache-policy
target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
target triple = "wasm32-unknown-unknown-wasm"
define void @globalfunc() #0 {
entry:
ret void
}

2
lld/wasm/LTO.cpp
View File

@ -142,7 +142,7 @@ std::vector<StringRef> BitcodeCompiler::compile() {
cache));
if (!config->thinLTOCacheDir.empty())
pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy);
pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy, files);
std::vector<StringRef> ret;
for (unsigned i = 0; i != maxTasks; ++i) {

10
llvm/include/llvm/Support/CachePruning.h
View File

@ -15,6 +15,7 @@
#define LLVM_SUPPORT_CACHEPRUNING_H
#include "llvm/ADT/Optional.h"
#include "llvm/Support/MemoryBuffer.h"
#include <chrono>
namespace llvm {
@ -70,11 +71,16 @@ Expected<CachePruningPolicy> parseCachePruningPolicy(StringRef PolicyStr);
/// Peform pruning using the supplied policy, returns true if pruning
/// occurred, i.e. if Policy.Interval was expired.
///
/// Check whether cache pruning happens using the supplied policy, adds a
/// ThinLTO warning if cache_size_bytes or cache_size_files is too small for the
/// current link job. The warning recommends the user to consider adjusting
/// --thinlto-cache-policy.
///
/// As a safeguard against data loss if the user specifies the wrong directory
/// as their cache directory, this function will ignore files not matching the
/// pattern "llvmcache-*".
bool pruneCache(StringRef Path, CachePruningPolicy Policy);
bool pruneCache(StringRef Path, CachePruningPolicy Policy,
const std::vector<std::unique_ptr<MemoryBuffer>> &Files = {});
} // namespace llvm
#endif

2
llvm/lib/LTO/ThinLTOCodeGenerator.cpp
View File

@ -1217,7 +1217,7 @@ void ThinLTOCodeGenerator::run() {
}
}
pruneCache(CacheOptions.Path, CacheOptions.Policy);
pruneCache(CacheOptions.Path, CacheOptions.Policy, ProducedBinaries);
// If statistics were requested, print them out now.
if (llvm::AreStatisticsEnabled())

28
llvm/lib/Support/CachePruning.cpp
View File

@ -17,6 +17,7 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
#define DEBUG_TYPE "cache-pruning"
@ -141,7 +142,8 @@ llvm::parseCachePruningPolicy(StringRef PolicyStr) {
}
/// Prune the cache of files that haven't been accessed in a long time.
bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy,
const std::vector<std::unique_ptr<MemoryBuffer>> &Files) {
using namespace std::chrono;
if (Path.empty())
@ -258,6 +260,17 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
++FileInfo;
};
// files.size() is greater the number of inputs by one. However, a timestamp
// file is created and stored in the cache directory if --thinlto-cache-policy
// option is used. Therefore, files.size() is used as ActualNums.
const size_t ActualNums = Files.size();
if (Policy.MaxSizeFiles && ActualNums > Policy.MaxSizeFiles)
WithColor::warning()
<< "ThinLTO cache pruning happens since the number of created files ("
<< ActualNums << ") exceeds the maximum number of files ("
<< Policy.MaxSizeFiles
<< "); consider adjusting --thinlto-cache-policy\n";
// Prune for number of files.
if (Policy.MaxSizeFiles)
while (NumFiles > Policy.MaxSizeFiles)
@ -285,6 +298,19 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
<< Policy.MaxSizePercentageOfAvailableSpace << "%, "
<< Policy.MaxSizeBytes << " bytes\n");
size_t ActualSizes = 0;
for (const auto &File : Files)
if (File)
ActualSizes += File->getBufferSize();
if (ActualSizes > TotalSizeTarget)
WithColor::warning()
<< "ThinLTO cache pruning happens since the total size of the cache "
"files consumed by the current link job ("
<< ActualSizes << " bytes) exceeds maximum cache size ("
<< TotalSizeTarget
<< " bytes); consider adjusting --thinlto-cache-policy\n";
// Remove the oldest accessed files first, till we get below the threshold.
while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
RemoveCacheFile();