shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[clang] Introduce ModuleCache::write() (#188877)

Browse Source 
This introduces new `ModuleCache` interface for writing PCM files.
Together with #188876, this will enable adding a caching layer into the
`InProcessModuleCache` implementation, hopefully reducing IO cost.
Moreover, this makes it super explicit that the PCM is written before
its timestamp, which is an important invariant that we've broken before.
This commit is contained in:
Jan Svoboda 2026-04-01 13:00:46 -07:00 committed by GitHub
parent 95a76886c1
commit 75c6f4791c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 123 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
clang/include/clang/Basic/DiagnosticCommonKinds.td
View File

@ -103,6 +103,7 @@ def err_deleted_non_function : Error<
"only functions can have deleted definitions">;
def err_module_not_found : Error<"module '%0' not found">, DefaultFatal;
def err_module_not_built : Error<"could not build module '%0'">, DefaultFatal;
def err_module_not_written : Error<"could not write module file for '%0' to '%1': %2">, DefaultFatal;
def err_module_build_disabled: Error<
"module '%0' is needed but has not been provided, and implicit use of module "
"files is disabled">, DefaultFatal;

10
clang/include/clang/Frontend/CompilerInstance.h
View File

@ -937,12 +937,14 @@ public:
std::optional<ThreadSafeCloneConfig> ThreadSafeConfig = std::nullopt);
/// Compile a module file for the given module, using the options
/// provided by the importing compiler instance. Returns true if the module
/// was built without errors.
/// provided by the importing compiler instance. Returns the PCM file in
/// a buffer.
// FIXME: This should be private, but it's called from static non-member
// functions in the implementation file.
bool compileModule(SourceLocation ImportLoc, StringRef ModuleName,
StringRef ModuleFileName, CompilerInstance &Instance);
std::unique_ptr<llvm::MemoryBuffer> compileModule(SourceLocation ImportLoc,
StringRef ModuleName,
StringRef ModuleFileName,
CompilerInstance &Instance);
ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path,
Module::NameVisibilityKind Visibility,

12
clang/include/clang/Frontend/FrontendActions.h
View File

@ -114,6 +114,15 @@ public:
};
class GenerateModuleAction : public ASTFrontendAction {
public:
/// When \c OS is non-null, uses it for outputting the PCM file instead of
/// automatically creating an output file.
explicit GenerateModuleAction(std::unique_ptr<raw_pwrite_stream> OS = nullptr)
: OS(std::move(OS)) {}
private:
std::unique_ptr<raw_pwrite_stream> OS;
virtual std::unique_ptr<raw_pwrite_stream>
CreateOutputFile(CompilerInstance &CI, StringRef InFile) = 0;
@ -145,6 +154,9 @@ protected:
};
class GenerateModuleFromModuleMapAction : public GenerateModuleAction {
public:
using GenerateModuleAction::GenerateModuleAction;
private:
bool BeginSourceFileAction(CompilerInstance &CI) override;

9
clang/include/clang/Serialization/ModuleCache.h
View File

@ -13,10 +13,12 @@
#include <ctime>
#include <memory>
#include <system_error>
namespace llvm {
class AdvisoryLock;
class MemoryBuffer;
class MemoryBufferRef;
} // namespace llvm
namespace clang {
@ -54,7 +56,9 @@ public:
virtual InMemoryModuleCache &getInMemoryModuleCache() = 0;
virtual const InMemoryModuleCache &getInMemoryModuleCache() const = 0;
// TODO: Virtualize writing PCM files.
/// Write the PCM contents to the given path in the module cache.
virtual std::error_code write(StringRef Path,
llvm::MemoryBufferRef Buffer) = 0;
virtual Expected<std::unique_ptr<llvm::MemoryBuffer>>
read(StringRef FileName, off_t &Size, time_t &ModTime) = 0;
@ -71,6 +75,9 @@ std::shared_ptr<ModuleCache> createCrossProcessModuleCache();
/// Shared implementation of `ModuleCache::maybePrune()`.
void maybePruneImpl(StringRef Path, time_t PruneInterval, time_t PruneAfter);
/// Shared implementation of `ModuleCache::write()`.
std::error_code writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer);
/// Shared implementation of `ModuleCache::read()`.
Expected<std::unique_ptr<llvm::MemoryBuffer>>
readImpl(StringRef FileName, off_t &Size, time_t &ModTime);

9
clang/lib/DependencyScanning/InProcessModuleCache.cpp
View File

@ -134,6 +134,15 @@ public:
return InMemory;
}
std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer) override {
// This is a compiler-internal input/output, let's bypass the sandbox.
auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
// FIXME: This could use an in-memory cache to avoid IO, and only write to
// disk at the end of the scan.
return writeImpl(Path, Buffer);
}
Expected<std::unique_ptr<llvm::MemoryBuffer>>
read(StringRef FileName, off_t &Size, time_t &ModTime) override {
// This is a compiler-internal input/output, let's bypass the sandbox.

61
clang/lib/Frontend/CompilerInstance.cpp
View File

@ -56,6 +56,7 @@
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/SmallVectorMemoryBuffer.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/VirtualFileSystem.h"
@ -1236,10 +1237,10 @@ public:
};
} // namespace
bool CompilerInstance::compileModule(SourceLocation ImportLoc,
StringRef ModuleName,
StringRef ModuleFileName,
CompilerInstance &Instance) {
std::unique_ptr<llvm::MemoryBuffer>
CompilerInstance::compileModule(SourceLocation ImportLoc, StringRef ModuleName,
StringRef ModuleFileName,
CompilerInstance &Instance) {
PrettyStackTraceBuildModule CrashInfo(ModuleName, ModuleFileName);
llvm::TimeTraceScope TimeScope("Module Compile", ModuleName);
@ -1248,18 +1249,22 @@ bool CompilerInstance::compileModule(SourceLocation ImportLoc,
if (getModuleCache().getInMemoryModuleCache().isPCMFinal(ModuleFileName)) {
getDiagnostics().Report(ImportLoc, diag::err_module_rebuild_finalized)
<< ModuleName;
return false;
return nullptr;
}
getDiagnostics().Report(ImportLoc, diag::remark_module_build)
<< ModuleName << ModuleFileName;
SmallString<0> Buffer;
// Execute the action to actually build the module in-place. Use a separate
// thread so that we get a stack large enough.
bool Crashed = !llvm::CrashRecoveryContext().RunSafelyOnNewStack(
[&]() {
auto OS = std::make_unique<llvm::raw_svector_ostream>(Buffer);
std::unique_ptr<FrontendAction> Action =
std::make_unique<GenerateModuleFromModuleMapAction>();
std::make_unique<GenerateModuleFromModuleMapAction>(std::move(OS));
if (auto WrapGenModuleAction = Instance.getGenModuleActionWrapper())
Action = WrapGenModuleAction(Instance.getFrontendOpts(),
@ -1295,10 +1300,17 @@ bool CompilerInstance::compileModule(SourceLocation ImportLoc,
setBuildGlobalModuleIndex(true);
}
// If \p AllowPCMWithCompilerErrors is set return 'success' even if errors
if (Crashed)
return nullptr;
// Unless \p AllowPCMWithCompilerErrors is set, return 'failure' if errors
// occurred.
return !Instance.getDiagnostics().hasErrorOccurred() ||
Instance.getFrontendOpts().AllowPCMWithCompilerErrors;
if (Instance.getDiagnostics().hasErrorOccurred() &&
!Instance.getFrontendOpts().AllowPCMWithCompilerErrors)
return nullptr;
return std::make_unique<llvm::SmallVectorMemoryBuffer>(
std::move(Buffer), Instance.getFrontendOpts().OutputFile);
}
static OptionalFileEntryRef getPublicModuleMap(FileEntryRef File,
@ -1440,13 +1452,17 @@ static bool compileModuleImpl(CompilerInstance &ImportingInstance,
SourceLocation ImportLoc,
SourceLocation ModuleNameLoc, Module *Module,
ModuleFileName ModuleFileName) {
std::unique_ptr<llvm::MemoryBuffer> Buffer;
{
auto Instance = ImportingInstance.cloneForModuleCompile(
ModuleNameLoc, Module, ModuleFileName);
if (!ImportingInstance.compileModule(ModuleNameLoc,
Module->getTopLevelModuleName(),
ModuleFileName, *Instance)) {
Buffer = ImportingInstance.compileModule(ModuleNameLoc,
Module->getTopLevelModuleName(),
ModuleFileName, *Instance);
if (!Buffer) {
ImportingInstance.getDiagnostics().Report(ModuleNameLoc,
diag::err_module_not_built)
<< Module->Name << SourceRange(ImportLoc, ModuleNameLoc);
@ -1454,6 +1470,16 @@ static bool compileModuleImpl(CompilerInstance &ImportingInstance,
}
}
std::error_code EC =
ImportingInstance.getModuleCache().write(ModuleFileName, *Buffer);
if (EC) {
ImportingInstance.getDiagnostics().Report(ModuleNameLoc,
diag::err_module_not_written)
<< Module->Name << ModuleFileName << EC.message()
<< SourceRange(ImportLoc, ModuleNameLoc);
return false;
}
// The module is built successfully, we can update its timestamp now.
if (ImportingInstance.getPreprocessor()
.getHeaderSearchInfo()
@ -2195,8 +2221,9 @@ void CompilerInstance::createModuleFromSource(SourceLocation ImportLoc,
// output is nondeterministic (as .pcm files refer to each other by name).
// Can this affect the output in any way?
SmallString<128> ModuleFileName;
int FD;
if (std::error_code EC = llvm::sys::fs::createTemporaryFile(
CleanModuleName, "pcm", ModuleFileName)) {
CleanModuleName, "pcm", FD, ModuleFileName)) {
getDiagnostics().Report(ImportLoc, diag::err_fe_unable_to_open_output)
<< ModuleFileName << EC.message();
return;
@ -2224,12 +2251,14 @@ void CompilerInstance::createModuleFromSource(SourceLocation ImportLoc,
Other->DeleteBuiltModules = false;
// Build the module, inheriting any modules that we've built locally.
bool Success = compileModule(ImportLoc, ModuleName, ModuleFileName, *Other);
std::unique_ptr<llvm::MemoryBuffer> Buffer =
compileModule(ImportLoc, ModuleName, ModuleFileName, *Other);
BuiltModules = std::move(Other->BuiltModules);
if (Success) {
if (Buffer) {
llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
BuiltModules[std::string(ModuleName)] = std::string(ModuleFileName);
OS << Buffer->getBuffer();
llvm::sys::RemoveFileOnSignal(ModuleFileName);
}
}

3
clang/lib/Frontend/FrontendActions.cpp
View File

@ -188,7 +188,8 @@ bool GeneratePCHAction::BeginSourceFileAction(CompilerInstance &CI) {
std::vector<std::unique_ptr<ASTConsumer>>
GenerateModuleAction::CreateMultiplexConsumer(CompilerInstance &CI,
StringRef InFile) {
std::unique_ptr<raw_pwrite_stream> OS = CreateOutputFile(CI, InFile);
if (!OS)
OS = CreateOutputFile(CI, InFile);
if (!OS)
return {};

40
clang/lib/Serialization/ModuleCache.cpp
View File

@ -102,6 +102,39 @@ void clang::maybePruneImpl(StringRef Path, time_t PruneInterval,
}
}
std::error_code clang::writeImpl(StringRef Path, llvm::MemoryBufferRef Buffer) {
StringRef Extension = llvm::sys::path::extension(Path);
SmallString<128> ModelPath = StringRef(Path).drop_back(Extension.size());
ModelPath += "-%%%%%%%%";
ModelPath += Extension;
ModelPath += ".tmp";
std::error_code EC;
int FD;
SmallString<128> TmpPath;
if ((EC = llvm::sys::fs::createUniqueFile(ModelPath, FD, TmpPath))) {
if (EC != std::errc::no_such_file_or_directory)
return EC;
StringRef Dir = llvm::sys::path::parent_path(Path);
if (std::error_code InnerEC = llvm::sys::fs::create_directories(Dir))
return InnerEC;
if ((EC = llvm::sys::fs::createUniqueFile(ModelPath, FD, TmpPath)))
return EC;
}
{
llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
OS << Buffer.getBuffer();
}
if ((EC = llvm::sys::fs::rename(TmpPath, Path)))
return EC;
return {};
}
Expected<std::unique_ptr<llvm::MemoryBuffer>>
clang::readImpl(StringRef FileName, off_t &Size, time_t &ModTime) {
Expected<llvm::sys::fs::file_t> FD =
@ -182,6 +215,13 @@ public:
return InMemory;
}
std::error_code write(StringRef Path, llvm::MemoryBufferRef Buffer) override {
// This is a compiler-internal input/output, let's bypass the sandbox.
auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
return writeImpl(Path, Buffer);
}
Expected<std::unique_ptr<llvm::MemoryBuffer>>
read(StringRef FileName, off_t &Size, time_t &ModTime) override {
// This is a compiler-internal input/output, let's bypass the sandbox.