shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
Jan Svoboda 983269bee7
[clang] Expose normalized module cache path in HeaderSearch (#185746) 
Previously, the normalized module cache path was only accessible via
`HeaderSearch::getSpecificModuleCachePath()` which may or may not also
contain the context hash. Clients would need to parse the result to
learn the normalized module cache path. What `ASTWriter` does instead is
normalize the as-written module cache path redundantly.

Instead, this PR exposes the normalized module cache path in the
`HeaderSearch` interface and moves the computation of specific module
cache path into the clangLex library.

This is motivated by another patch that would've needed to redundantly
perform the module cache path canonicalization or parse the specific
module cache path.
2026-03-12 13:52:42 -07:00

833 lines
33 KiB
C++
Raw Blame History

//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/DiagnosticSerialization.h"
#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
#include "clang/DependencyScanning/DependencyScanningService.h"
#include "clang/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Driver/Driver.h"
#include "clang/Frontend/FrontendActions.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Support/AdvisoryLock.h"
#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Host.h"
#include <mutex>
#include <thread>
using namespace clang;
using namespace dependencies;
namespace {
/// Forwards the gatherered dependencies to the consumer.
class DependencyConsumerForwarder : public DependencyFileGenerator {
public:
DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
StringRef WorkingDirectory, DependencyConsumer &C)
: DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
Opts(std::move(Opts)), C(C) {}
void finishedMainFile(DiagnosticsEngine &Diags) override {
C.handleDependencyOutputOpts(*Opts);
llvm::SmallString<256> CanonPath;
for (const auto &File : getDependencies()) {
CanonPath = File;
llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
llvm::sys::path::make_absolute(WorkingDirectory, CanonPath);
C.handleFileDependency(CanonPath);
}
}
private:
StringRef WorkingDirectory;
std::unique_ptr<DependencyOutputOptions> Opts;
DependencyConsumer &C;
};
static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
const HeaderSearchOptions &ExistingHSOpts,
DiagnosticsEngine *Diags,
const LangOptions &LangOpts) {
if (LangOpts.Modules) {
if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
if (Diags) {
Diags->Report(diag::warn_pch_vfsoverlay_mismatch);
auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
if (VFSOverlays.empty()) {
Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;
} else {
std::string Files = llvm::join(VFSOverlays, "\n");
Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;
}
};
VFSNote(0, HSOpts.VFSOverlayFiles);
VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
}
}
}
return false;
}
using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
/// A listener that collects the imported modules and the input
/// files. While visiting, collect vfsoverlays and file inputs that determine
/// whether prebuilt modules fully resolve in stable directories.
class PrebuiltModuleListener : public ASTReaderListener {
public:
PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
llvm::SmallVector<std::string> &NewModuleFiles,
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
const HeaderSearchOptions &HSOpts,
const LangOptions &LangOpts, DiagnosticsEngine &Diags,
const ArrayRef<StringRef> StableDirs)
: PrebuiltModuleFiles(PrebuiltModuleFiles),
NewModuleFiles(NewModuleFiles),
PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
bool needsImportVisitation() const override { return true; }
bool needsInputFileVisitation() override { return true; }
bool needsSystemInputFileVisitation() override { return true; }
/// Accumulate the modules are transitively depended on by the initial
/// prebuilt module.
void visitImport(StringRef ModuleName, StringRef Filename) override {
if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
NewModuleFiles.push_back(Filename.str());
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
if (auto It = PrebuiltModulesASTMap.find(CurrentFile);
It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
PrebuiltModule.addDependent(It->getKey());
}
/// For each input file discovered, check whether it's external path is in a
/// stable directory. Traversal is stopped if the current module is not
/// considered stable.
bool visitInputFileAsRequested(StringRef FilenameAsRequested,
StringRef Filename, bool isSystem,
bool isOverridden, time_t StoredTime,
bool isExplicitModule) override {
if (StableDirs.empty())
return false;
auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
(!PrebuiltEntryIt->second.isInStableDir()))
return false;
PrebuiltEntryIt->second.setInStableDir(
isPathInStableDir(StableDirs, Filename));
return PrebuiltEntryIt->second.isInStableDir();
}
/// Update which module that is being actively traversed.
void visitModuleFile(StringRef Filename,
serialization::ModuleKind Kind) override {
// If the CurrentFile is not
// considered stable, update any of it's transitive dependents.
auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
!PrebuiltEntryIt->second.isInStableDir())
PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
PrebuiltModulesASTMap);
CurrentFile = Filename;
}
/// Check the header search options for a given module when considering
/// if the module comes from stable directories.
bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
StringRef ModuleFilename, StringRef ContextHash,
bool Complain) override {
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
if (PrebuiltModule.isInStableDir())
PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts));
return false;
}
/// Accumulate vfsoverlays used to build these prebuilt modules.
bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
bool Complain) override {
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
PrebuiltModule.setVFS(
llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
return checkHeaderSearchPaths(
HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
}
private:
PrebuiltModuleFilesT &PrebuiltModuleFiles;
llvm::SmallVector<std::string> &NewModuleFiles;
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
const HeaderSearchOptions &ExistingHSOpts;
const LangOptions &ExistingLangOpts;
DiagnosticsEngine &Diags;
std::string CurrentFile;
const ArrayRef<StringRef> StableDirs;
};
/// Visit the given prebuilt module and collect all of the modules it
/// transitively imports and contributing input files.
static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
CompilerInstance &CI,
PrebuiltModuleFilesT &ModuleFiles,
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
DiagnosticsEngine &Diags,
const ArrayRef<StringRef> StableDirs) {
// List of module files to be processed.
llvm::SmallVector<std::string> Worklist;
PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
CI.getHeaderSearchOpts(), CI.getLangOpts(),
Diags, StableDirs);
Listener.visitModuleFile(PrebuiltModuleFilename,
serialization::MK_ExplicitModule);
if (ASTReader::readASTFileControlBlock(
PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),
CI.getPCHContainerReader(),
/*FindModuleFileExtensions=*/false, Listener,
/*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))
return true;
while (!Worklist.empty()) {
Listener.visitModuleFile(Worklist.back(), serialization::MK_ExplicitModule);
if (ASTReader::readASTFileControlBlock(
Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
CI.getPCHContainerReader(),
/*FindModuleFileExtensions=*/false, Listener,
/*ValidateDiagnosticOptions=*/false))
return true;
}
return false;
}
/// Transform arbitrary file name into an object-like file name.
static std::string makeObjFileName(StringRef FileName) {
SmallString<128> ObjFileName(FileName);
llvm::sys::path::replace_extension(ObjFileName, "o");
return std::string(ObjFileName);
}
/// Deduce the dependency target based on the output file and input files.
static std::string
deduceDepTarget(const std::string &OutputFile,
const SmallVectorImpl<FrontendInputFile> &InputFiles) {
if (OutputFile != "-")
return OutputFile;
if (InputFiles.empty() || !InputFiles.front().isFile())
return "clang-scan-deps\\ dependency";
return makeObjFileName(InputFiles.front().getFile());
}
// Clang implements -D and -U by splatting text into a predefines buffer. This
// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
// define the same macro, or adding C++ style comments before the macro name.
//
// This function checks that the first non-space characters in the macro
// obviously form an identifier that can be uniqued on without lexing. Failing
// to do this could lead to changing the final definition of a macro.
//
// We could set up a preprocessor and actually lex the name, but that's very
// heavyweight for a situation that will almost never happen in practice.
static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
StringRef Name = Macro.split("=").first.ltrim(" \t");
std::size_t I = 0;
auto FinishName = [&]() -> std::optional<StringRef> {
StringRef SimpleName = Name.slice(0, I);
if (SimpleName.empty())
return std::nullopt;
return SimpleName;
};
for (; I != Name.size(); ++I) {
switch (Name[I]) {
case '(': // Start of macro parameter list
case ' ': // End of macro name
case '\t':
return FinishName();
case '_':
continue;
default:
if (llvm::isAlnum(Name[I]))
continue;
return std::nullopt;
}
}
return FinishName();
}
} // namespace
void dependencies::canonicalizeDefines(PreprocessorOptions &PPOpts) {
using MacroOpt = std::pair<StringRef, std::size_t>;
std::vector<MacroOpt> SimpleNames;
SimpleNames.reserve(PPOpts.Macros.size());
std::size_t Index = 0;
for (const auto &M : PPOpts.Macros) {
auto SName = getSimpleMacroName(M.first);
// Skip optimizing if we can't guarantee we can preserve relative order.
if (!SName)
return;
SimpleNames.emplace_back(*SName, Index);
++Index;
}
llvm::stable_sort(SimpleNames, llvm::less_first());
// Keep the last instance of each macro name by going in reverse
auto NewEnd = std::unique(
SimpleNames.rbegin(), SimpleNames.rend(),
[](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
SimpleNames.erase(SimpleNames.begin(), NewEnd.base());
// Apply permutation.
decltype(PPOpts.Macros) NewMacros;
NewMacros.reserve(SimpleNames.size());
for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
std::size_t OriginalIndex = SimpleNames[I].second;
// We still emit undefines here as they may be undefining a predefined macro
NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex]));
}
std::swap(PPOpts.Macros, NewMacros);
}
namespace {
class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
DependencyScanningWorkerFilesystem *DepFS;
public:
ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) {
auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS);
if (DFS) {
assert(!DepFS && "Found multiple scanning VFSs");
DepFS = DFS;
}
});
assert(DepFS && "Did not find scanning VFS");
}
std::unique_ptr<DependencyDirectivesGetter>
cloneFor(FileManager &FileMgr) override {
return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr);
}
std::optional<ArrayRef<dependency_directives_scan::Directive>>
operator()(FileEntryRef File) override {
return DepFS->getDirectiveTokens(File.getName());
}
};
/// Sanitize diagnostic options for dependency scan.
void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
// Don't print 'X warnings and Y errors generated'.
DiagOpts.ShowCarets = false;
// Don't write out diagnostic file.
DiagOpts.DiagnosticSerializationFile.clear();
// Don't emit warnings except for scanning specific warnings.
// TODO: It would be useful to add a more principled way to ignore all
// warnings that come from source code. The issue is that we need to
// ignore warnings that could be surpressed by
// `#pragma clang diagnostic`, while still allowing some scanning
// warnings for things we're not ready to turn into errors yet.
// See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) {
return llvm::StringSwitch<bool>(Warning)
.Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false)
.StartsWith("no-error=", false)
.Default(true);
});
}
} // namespace
std::unique_ptr<DiagnosticOptions>
dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {
std::vector<const char *> CLI;
for (const std::string &Arg : CommandLine)
CLI.push_back(Arg.c_str());
auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
sanitizeDiagOpts(*DiagOpts);
return DiagOpts;
}
DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
ArrayRef<std::string> CommandLine,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
llvm::transform(CommandLine, CCommandLine.begin(),
[](const std::string &Str) { return Str.c_str(); });
DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);
sanitizeDiagOpts(*DiagOpts);
DiagEngine = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC,
/*ShouldOwnClient=*/false);
}
std::unique_ptr<CompilerInvocation>
dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,
DiagnosticsEngine &Diags) {
llvm::opt::ArgStringList Argv;
for (const std::string &Str : ArrayRef(CommandLine).drop_front())
Argv.push_back(Str.c_str());
auto Invocation = std::make_unique<CompilerInvocation>();
if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) {
// FIXME: Should we just go on like cc1_main does?
return nullptr;
}
return Invocation;
}
void dependencies::initializeScanCompilerInstance(
CompilerInstance &ScanInstance,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
ScanInstance.setBuildingModule(false);
ScanInstance.createVirtualFileSystem(FS, DiagConsumer);
ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
ScanInstance.createFileManager();
ScanInstance.createSourceManager();
// Use DepFS for getting the dependency directives if requested to do so.
if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) {
DepFS->resetBypassedPathPrefix();
SmallString<256> ModulesCachePath;
normalizeModuleCachePath(ScanInstance.getFileManager(),
ScanInstance.getHeaderSearchOpts().ModuleCachePath,
ModulesCachePath);
if (!ModulesCachePath.empty())
DepFS->setBypassedPathPrefix(ModulesCachePath);
ScanInstance.setDependencyDirectivesGetter(
std::make_unique<ScanningDependencyDirectivesGetter>(
ScanInstance.getFileManager()));
}
}
std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation(
const CompilerInvocation &Invocation,
const DependencyScanningService &Service,
DependencyActionController &Controller) {
auto ScanInvocation = std::make_shared<CompilerInvocation>(Invocation);
sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts());
ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
true;
if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
Service.getOpts().BuildSessionTimestamp;
ScanInvocation->getFrontendOpts().DisableFree = false;
ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
ScanInvocation->getFrontendOpts().GenReducedBMI = false;
ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
// This will prevent us compiling individual modules asynchronously since
// FileManager is not thread-safe, but it does improve performance for now.
ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
any(Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
// Consider different header search and diagnostic options to create
// different modules. This avoids the unsound aliasing of module PCMs.
//
// TODO: Implement diagnostic bucketing to reduce the impact of strict
// context hashing.
ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
true;
ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
// Ensure that the scanner does not create new dependency collectors,
// and thus won't write out the extra '.d' files to disk.
ScanInvocation->getDependencyOutputOpts() = {};
Controller.initializeScanInvocation(*ScanInvocation);
return ScanInvocation;
}
llvm::SmallVector<StringRef>
dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) {
// Create a collection of stable directories derived from the ScanInstance
// for determining whether module dependencies would fully resolve from
// those directories.
llvm::SmallVector<StringRef> StableDirs;
const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot))
StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
return StableDirs;
}
std::optional<PrebuiltModulesAttrsMap>
dependencies::computePrebuiltModulesASTMap(
CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {
// Store a mapping of prebuilt module files and their properties like header
// search options. This will prevent the implicit build to create duplicate
// modules and will force reuse of the existing prebuilt module files
// instead.
PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
if (visitPrebuiltModule(
ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs))
return {};
return PrebuiltModulesASTMap;
}
std::unique_ptr<DependencyOutputOptions>
dependencies::createDependencyOutputOptions(
const CompilerInvocation &Invocation) {
auto Opts = std::make_unique<DependencyOutputOptions>(
Invocation.getDependencyOutputOpts());
// We need at least one -MT equivalent for the generator of make dependency
// files to work.
if (Opts->Targets.empty())
Opts->Targets = {deduceDepTarget(Invocation.getFrontendOpts().OutputFile,
Invocation.getFrontendOpts().Inputs)};
Opts->IncludeSystemHeaders = true;
return Opts;
}
std::shared_ptr<ModuleDepCollector>
dependencies::initializeScanInstanceDependencyCollector(
CompilerInstance &ScanInstance,
std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
StringRef WorkingDirectory, DependencyConsumer &Consumer,
DependencyScanningService &Service, CompilerInvocation &Inv,
DependencyActionController &Controller,
PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
llvm::SmallVector<StringRef> &StableDirs) {
std::shared_ptr<ModuleDepCollector> MDC;
switch (Service.getOpts().Format) {
case ScanningOutputFormat::Make:
ScanInstance.addDependencyCollector(
std::make_shared<DependencyConsumerForwarder>(
std::move(DepOutputOpts), WorkingDirectory, Consumer));
break;
case ScanningOutputFormat::P1689:
case ScanningOutputFormat::Full:
MDC = std::make_shared<ModuleDepCollector>(
Service, std::move(DepOutputOpts), ScanInstance, Consumer, Controller,
Inv, std::move(PrebuiltModulesASTMap), StableDirs);
ScanInstance.addDependencyCollector(MDC);
break;
}
return MDC;
}
/// Manages (and terminates) the asynchronous compilation of modules.
class AsyncModuleCompiles {
std::mutex Mutex;
bool Stop = false;
// FIXME: Have the service own a thread pool and use that instead.
std::vector<std::thread> Compiles;
public:
/// Registers the module compilation, unless this instance is about to be
/// destroyed.
void add(llvm::unique_function<void()> Compile) {
std::lock_guard<std::mutex> Lock(Mutex);
if (!Stop)
Compiles.emplace_back(std::move(Compile));
}
~AsyncModuleCompiles() {
{
// Prevent registration of further module compiles.
std::lock_guard<std::mutex> Lock(Mutex);
Stop = true;
}
// Wait for outstanding module compiles to finish.
for (std::thread &Compile : Compiles)
Compile.join();
}
};
struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
DependencyScanningService &Service;
DependencyActionController &Controller;
AsyncModuleCompiles &Compiles;
SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
DependencyActionController &Controller,
AsyncModuleCompiles &Compiles)
: Service(Service), Controller(Controller), Compiles(Compiles) {}
bool BeginSourceFileAction(CompilerInstance &CI) override;
};
/// The preprocessor callback that takes care of initiating an asynchronous
/// module compilation if needed.
struct AsyncModuleCompile : PPCallbacks {
CompilerInstance &CI;
DependencyScanningService &Service;
DependencyActionController &Controller;
AsyncModuleCompiles &Compiles;
AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
DependencyActionController &Controller,
AsyncModuleCompiles &Compiles)
: CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {}
void moduleLoadSkipped(Module *M) override {
M = M->getTopLevelModule();
HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
ModuleCache &ModCache = CI.getModuleCache();
std::string ModuleFileName = HS.getCachedModuleFileName(M);
uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName);
// Someone else already built/validated the PCM.
if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
return;
if (!CI.getASTReader())
CI.createASTReader();
SmallVector<ASTReader::ImportedModule, 0> Imported;
// Only calling ReadASTCore() to avoid the expensive eager deserialization
// of the clang::Module objects in ReadAST().
// FIXME: Consider doing this in the new thread depending on how expensive
// the read turns out to be.
switch (CI.getASTReader()->ReadASTCore(
ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(),
nullptr, Imported, {}, {}, {},
ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
case ASTReader::Success:
// We successfully read a valid, up-to-date PCM.
// FIXME: This could update the timestamp. Regular calls to
// ASTReader::ReadAST() would do so unless they encountered corrupted
// AST block, corrupted extension block, or did not read the expected
// top-level module.
return;
case ASTReader::OutOfDate:
case ASTReader::Missing:
// The most interesting case.
break;
default:
// Let the regular scan diagnose this.
return;
}
ModCache.prepareForGetLock(ModuleFileName);
auto Lock = ModCache.getLock(ModuleFileName);
bool Owned;
llvm::Error LockErr = Lock->tryLock().moveInto(Owned);
// Someone else is building the PCM right now.
if (!LockErr && !Owned)
return;
// We should build the PCM.
IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
Service.getSharedCache(), Service.getOpts().MakeVFS());
VFS = createVFSFromCompilerInvocation(CI.getInvocation(),
CI.getDiagnostics(), std::move(VFS));
auto DC = std::make_unique<DiagnosticConsumer>();
auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries());
CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
std::move(MC));
auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
CloneConfig);
auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
CloneConfig);
auto ModController = Controller.clone();
// Note: This lock belongs to a module cache that might not outlive the
// thread. This works, because the in-process lock only refers to an object
// managed by the service, which does outlive the thread.
Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
ModCI2 = std::move(ModCI2), DC = std::move(DC),
ModController = std::move(ModController), Service = &Service,
Compiles = &Compiles] {
llvm::CrashRecoveryContext CRC;
(void)CRC.RunSafely([&] {
// Quickly discovers and compiles modules for the real scan below.
SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController,
*Compiles);
(void)ModCI1->ExecuteAction(Action1);
// The real scan below.
ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
GenerateModuleFromModuleMapAction Action2;
(void)ModCI2->ExecuteAction(Action2);
});
});
}
};
/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
/// modules asynchronously without blocking or importing them.
struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
DependencyScanningService &Service;
DependencyActionController &Controller;
AsyncModuleCompiles &Compiles;
SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
DependencyActionController &Controller,
AsyncModuleCompiles &Compiles)
: Service(Service), Controller(Controller), Compiles(Compiles) {}
bool BeginSourceFileAction(CompilerInstance &CI) override {
CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
CI.getPreprocessor().addPPCallbacks(std::make_unique<AsyncModuleCompile>(
CI, Service, Controller, Compiles));
return true;
}
};
bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
CompilerInstance &CI) {
CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
CI.getPreprocessor().addPPCallbacks(
std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles));
return true;
}
bool DependencyScanningAction::runInvocation(
std::string Executable,
std::unique_ptr<CompilerInvocation> OriginalInvocation,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
DiagnosticConsumer *DiagConsumer) {
// Making sure that we canonicalize the defines early to avoid unnecessary
// variants in both the scanner and in the resulting explicit command lines.
if (any(Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
if (Scanned) {
CompilerInstance &ScanInstance = *ScanInstanceStorage;
// Scanning runs once for the first -cc1 invocation in a chain of driver
// jobs. For any dependent jobs, reuse the scanning result and just
// update the new invocation.
// FIXME: to support multi-arch builds, each arch requires a separate scan
if (MDC)
MDC->applyDiscoveredDependencies(*OriginalInvocation);
if (!Controller.finalize(ScanInstance, *OriginalInvocation))
return false;
Consumer.handleBuildCommand(
{Executable, OriginalInvocation->getCC1CommandLine()});
return true;
}
Scanned = true;
// Create a compiler instance to handle the actual work.
auto ScanInvocation =
createScanCompilerInvocation(*OriginalInvocation, Service, Controller);
// Quickly discovers and compiles modules for the real scan below.
std::optional<AsyncModuleCompiles> AsyncCompiles;
if (Service.getOpts().AsyncScanModules) {
auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
auto ScanInstanceStorage = std::make_unique<CompilerInstance>(
std::make_shared<CompilerInvocation>(*ScanInvocation), PCHContainerOps,
std::move(ModCache));
CompilerInstance &ScanInstance = *ScanInstanceStorage;
DiagnosticConsumer DiagConsumer;
initializeScanCompilerInstance(ScanInstance, FS, &DiagConsumer, Service,
DepFS);
// FIXME: Do this only once.
SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
auto MaybePrebuiltModulesASTMap =
computePrebuiltModulesASTMap(ScanInstance, StableDirs);
if (!MaybePrebuiltModulesASTMap)
return false;
// Normally this would be handled by GeneratePCHAction
if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
ScanInstance.getLangOpts().CompilingPCH = true;
AsyncCompiles.emplace();
SingleTUWithAsyncModuleCompiles Action(Service, Controller, *AsyncCompiles);
(void)ScanInstance.ExecuteAction(Action);
}
auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
ScanInstanceStorage.emplace(std::move(ScanInvocation),
std::move(PCHContainerOps), std::move(ModCache));
CompilerInstance &ScanInstance = *ScanInstanceStorage;
initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
DepFS);
llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
auto MaybePrebuiltModulesASTMap =
computePrebuiltModulesASTMap(ScanInstance, StableDirs);
if (!MaybePrebuiltModulesASTMap)
return false;
auto DepOutputOpts = createDependencyOutputOptions(*OriginalInvocation);
MDC = initializeScanInstanceDependencyCollector(
ScanInstance, std::move(DepOutputOpts), WorkingDirectory, Consumer,
Service, *OriginalInvocation, Controller, *MaybePrebuiltModulesASTMap,
StableDirs);
if (ScanInstance.getDiagnostics().hasErrorOccurred())
return false;
if (!Controller.initialize(ScanInstance, *OriginalInvocation))
return false;
ReadPCHAndPreprocessAction Action;
const bool Result = ScanInstance.ExecuteAction(Action);
if (Result) {
if (MDC)
MDC->applyDiscoveredDependencies(*OriginalInvocation);
if (!Controller.finalize(ScanInstance, *OriginalInvocation))
return false;
Consumer.handleBuildCommand(
{Executable, OriginalInvocation->getCC1CommandLine()});
}
return Result;
}
Reference in New Issue View Git Blame Copy Permalink