shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/clang/lib/DependencyScanning/DependencyScannerImpl.cpp
Jan Svoboda 5b064a2c16
[clang][deps] Simplify by-module-name scan API (#184376) 
The by-module-name scanning APIs are fairly spread out. There's the main
`CompilerInstanceWithContext` class that provides a constructor,
`initialize()` and `computeDependencies()`. Then there's the
`DependencyScanningWorker` that optionally owns
`CompilerInstanceWithContext` and re-exposes two `initialize()` and one
`computeDependencies()` functions. Lastly, there's
`DependencyScanningTool` that again re-exposes two variants of
`initialize()` and one `computeDependencies()`.

The current setup makes it unnecessarily difficult to make changes to
these APIs (as observed in
https://github.com/swiftlang/llvm-project/pull/12453).

This PR makes `CompilerInstanceWithContext` standalone, and hides the
construct + initialize pattern behind a static factory function. This
makes it harder to misuse the API (forgetting to call `initialize()`,
calling it twice, etc.) and means changes now need to only touch single
class instead of three classes spread over multiple files.
2026-03-05 10:26:46 -08:00

808 lines
32 KiB
C++
Raw Blame History

//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "clang/DependencyScanning/DependencyScannerImpl.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/DiagnosticSerialization.h"
#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
#include "clang/DependencyScanning/DependencyScanningService.h"
#include "clang/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Driver/Driver.h"
#include "clang/Frontend/FrontendActions.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Support/AdvisoryLock.h"
#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Host.h"
#include <mutex>
#include <thread>
using namespace clang;
using namespace dependencies;
namespace {
/// Forwards the gatherered dependencies to the consumer.
class DependencyConsumerForwarder : public DependencyFileGenerator {
public:
DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
StringRef WorkingDirectory, DependencyConsumer &C)
: DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
Opts(std::move(Opts)), C(C) {}
void finishedMainFile(DiagnosticsEngine &Diags) override {
C.handleDependencyOutputOpts(*Opts);
llvm::SmallString<256> CanonPath;
for (const auto &File : getDependencies()) {
CanonPath = File;
llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
llvm::sys::path::make_absolute(WorkingDirectory, CanonPath);
C.handleFileDependency(CanonPath);
}
}
private:
StringRef WorkingDirectory;
std::unique_ptr<DependencyOutputOptions> Opts;
DependencyConsumer &C;
};
static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
const HeaderSearchOptions &ExistingHSOpts,
DiagnosticsEngine *Diags,
const LangOptions &LangOpts) {
if (LangOpts.Modules) {
if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
if (Diags) {
Diags->Report(diag::warn_pch_vfsoverlay_mismatch);
auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
if (VFSOverlays.empty()) {
Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;
} else {
std::string Files = llvm::join(VFSOverlays, "\n");
Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;
}
};
VFSNote(0, HSOpts.VFSOverlayFiles);
VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
}
}
}
return false;
}
using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
/// A listener that collects the imported modules and the input
/// files. While visiting, collect vfsoverlays and file inputs that determine
/// whether prebuilt modules fully resolve in stable directories.
class PrebuiltModuleListener : public ASTReaderListener {
public:
PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
llvm::SmallVector<std::string> &NewModuleFiles,
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
const HeaderSearchOptions &HSOpts,
const LangOptions &LangOpts, DiagnosticsEngine &Diags,
const ArrayRef<StringRef> StableDirs)
: PrebuiltModuleFiles(PrebuiltModuleFiles),
NewModuleFiles(NewModuleFiles),
PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
bool needsImportVisitation() const override { return true; }
bool needsInputFileVisitation() override { return true; }
bool needsSystemInputFileVisitation() override { return true; }
/// Accumulate the modules are transitively depended on by the initial
/// prebuilt module.
void visitImport(StringRef ModuleName, StringRef Filename) override {
if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
NewModuleFiles.push_back(Filename.str());
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
if (auto It = PrebuiltModulesASTMap.find(CurrentFile);
It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
PrebuiltModule.addDependent(It->getKey());
}
/// For each input file discovered, check whether it's external path is in a
/// stable directory. Traversal is stopped if the current module is not
/// considered stable.
bool visitInputFileAsRequested(StringRef FilenameAsRequested,
StringRef Filename, bool isSystem,
bool isOverridden, time_t StoredTime,
bool isExplicitModule) override {
if (StableDirs.empty())
return false;
auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
(!PrebuiltEntryIt->second.isInStableDir()))
return false;
PrebuiltEntryIt->second.setInStableDir(
isPathInStableDir(StableDirs, Filename));
return PrebuiltEntryIt->second.isInStableDir();
}
/// Update which module that is being actively traversed.
void visitModuleFile(StringRef Filename,
serialization::ModuleKind Kind) override {
// If the CurrentFile is not
// considered stable, update any of it's transitive dependents.
auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
!PrebuiltEntryIt->second.isInStableDir())
PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
PrebuiltModulesASTMap);
CurrentFile = Filename;
}
/// Check the header search options for a given module when considering
/// if the module comes from stable directories.
bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
StringRef ModuleFilename,
StringRef SpecificModuleCachePath,
bool Complain) override {
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
if (PrebuiltModule.isInStableDir())
PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts));
return false;
}
/// Accumulate vfsoverlays used to build these prebuilt modules.
bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
bool Complain) override {
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
PrebuiltModule.setVFS(
llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
return checkHeaderSearchPaths(
HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
}
private:
PrebuiltModuleFilesT &PrebuiltModuleFiles;
llvm::SmallVector<std::string> &NewModuleFiles;
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
const HeaderSearchOptions &ExistingHSOpts;
const LangOptions &ExistingLangOpts;
DiagnosticsEngine &Diags;
std::string CurrentFile;
const ArrayRef<StringRef> StableDirs;
};
/// Visit the given prebuilt module and collect all of the modules it
/// transitively imports and contributing input files.
static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
CompilerInstance &CI,
PrebuiltModuleFilesT &ModuleFiles,
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
DiagnosticsEngine &Diags,
const ArrayRef<StringRef> StableDirs) {
// List of module files to be processed.
llvm::SmallVector<std::string> Worklist;
PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
CI.getHeaderSearchOpts(), CI.getLangOpts(),
Diags, StableDirs);
Listener.visitModuleFile(PrebuiltModuleFilename,
serialization::MK_ExplicitModule);
if (ASTReader::readASTFileControlBlock(
PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),
CI.getPCHContainerReader(),
/*FindModuleFileExtensions=*/false, Listener,
/*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))
return true;
while (!Worklist.empty()) {
Listener.visitModuleFile(Worklist.back(), serialization::MK_ExplicitModule);
if (ASTReader::readASTFileControlBlock(
Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
CI.getPCHContainerReader(),
/*FindModuleFileExtensions=*/false, Listener,
/*ValidateDiagnosticOptions=*/false))
return true;
}
return false;
}
/// Transform arbitrary file name into an object-like file name.
static std::string makeObjFileName(StringRef FileName) {
SmallString<128> ObjFileName(FileName);
llvm::sys::path::replace_extension(ObjFileName, "o");
return std::string(ObjFileName);
}
/// Deduce the dependency target based on the output file and input files.
static std::string
deduceDepTarget(const std::string &OutputFile,
const SmallVectorImpl<FrontendInputFile> &InputFiles) {
if (OutputFile != "-")
return OutputFile;
if (InputFiles.empty() || !InputFiles.front().isFile())
return "clang-scan-deps\\ dependency";
return makeObjFileName(InputFiles.front().getFile());
}
// Clang implements -D and -U by splatting text into a predefines buffer. This
// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
// define the same macro, or adding C++ style comments before the macro name.
//
// This function checks that the first non-space characters in the macro
// obviously form an identifier that can be uniqued on without lexing. Failing
// to do this could lead to changing the final definition of a macro.
//
// We could set up a preprocessor and actually lex the name, but that's very
// heavyweight for a situation that will almost never happen in practice.
static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
StringRef Name = Macro.split("=").first.ltrim(" \t");
std::size_t I = 0;
auto FinishName = [&]() -> std::optional<StringRef> {
StringRef SimpleName = Name.slice(0, I);
if (SimpleName.empty())
return std::nullopt;
return SimpleName;
};
for (; I != Name.size(); ++I) {
switch (Name[I]) {
case '(': // Start of macro parameter list
case ' ': // End of macro name
case '\t':
return FinishName();
case '_':
continue;
default:
if (llvm::isAlnum(Name[I]))
continue;
return std::nullopt;
}
}
return FinishName();
}
} // namespace
void dependencies::canonicalizeDefines(PreprocessorOptions &PPOpts) {
using MacroOpt = std::pair<StringRef, std::size_t>;
std::vector<MacroOpt> SimpleNames;
SimpleNames.reserve(PPOpts.Macros.size());
std::size_t Index = 0;
for (const auto &M : PPOpts.Macros) {
auto SName = getSimpleMacroName(M.first);
// Skip optimizing if we can't guarantee we can preserve relative order.
if (!SName)
return;
SimpleNames.emplace_back(*SName, Index);
++Index;
}
llvm::stable_sort(SimpleNames, llvm::less_first());
// Keep the last instance of each macro name by going in reverse
auto NewEnd = std::unique(
SimpleNames.rbegin(), SimpleNames.rend(),
[](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
SimpleNames.erase(SimpleNames.begin(), NewEnd.base());
// Apply permutation.
decltype(PPOpts.Macros) NewMacros;
NewMacros.reserve(SimpleNames.size());
for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
std::size_t OriginalIndex = SimpleNames[I].second;
// We still emit undefines here as they may be undefining a predefined macro
NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex]));
}
std::swap(PPOpts.Macros, NewMacros);
}
namespace {
class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
DependencyScanningWorkerFilesystem *DepFS;
public:
ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) {
auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS);
if (DFS) {
assert(!DepFS && "Found multiple scanning VFSs");
DepFS = DFS;
}
});
assert(DepFS && "Did not find scanning VFS");
}
std::unique_ptr<DependencyDirectivesGetter>
cloneFor(FileManager &FileMgr) override {
return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr);
}
std::optional<ArrayRef<dependency_directives_scan::Directive>>
operator()(FileEntryRef File) override {
return DepFS->getDirectiveTokens(File.getName());
}
};
/// Sanitize diagnostic options for dependency scan.
void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
// Don't print 'X warnings and Y errors generated'.
DiagOpts.ShowCarets = false;
// Don't write out diagnostic file.
DiagOpts.DiagnosticSerializationFile.clear();
// Don't emit warnings except for scanning specific warnings.
// TODO: It would be useful to add a more principled way to ignore all
// warnings that come from source code. The issue is that we need to
// ignore warnings that could be surpressed by
// `#pragma clang diagnostic`, while still allowing some scanning
// warnings for things we're not ready to turn into errors yet.
// See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) {
return llvm::StringSwitch<bool>(Warning)
.Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false)
.StartsWith("no-error=", false)
.Default(true);
});
}
} // namespace
std::unique_ptr<DiagnosticOptions>
dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) {
std::vector<const char *> CLI;
for (const std::string &Arg : CommandLine)
CLI.push_back(Arg.c_str());
auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
sanitizeDiagOpts(*DiagOpts);
return DiagOpts;
}
DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts(
ArrayRef<std::string> CommandLine,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) {
std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
llvm::transform(CommandLine, CCommandLine.begin(),
[](const std::string &Str) { return Str.c_str(); });
DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);
sanitizeDiagOpts(*DiagOpts);
DiagEngine = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC,
/*ShouldOwnClient=*/false);
}
std::unique_ptr<CompilerInvocation>
dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine,
DiagnosticsEngine &Diags) {
llvm::opt::ArgStringList Argv;
for (const std::string &Str : ArrayRef(CommandLine).drop_front())
Argv.push_back(Str.c_str());
auto Invocation = std::make_unique<CompilerInvocation>();
if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) {
// FIXME: Should we just go on like cc1_main does?
return nullptr;
}
return Invocation;
}
void dependencies::initializeScanCompilerInstance(
CompilerInstance &ScanInstance,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service,
IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) {
ScanInstance.setBuildingModule(false);
ScanInstance.createVirtualFileSystem(FS, DiagConsumer);
ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
ScanInstance.createFileManager();
ScanInstance.createSourceManager();
// Use DepFS for getting the dependency directives if requested to do so.
if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) {
DepFS->resetBypassedPathPrefix();
SmallString<256> ModulesCachePath;
normalizeModuleCachePath(ScanInstance.getFileManager(),
ScanInstance.getHeaderSearchOpts().ModuleCachePath,
ModulesCachePath);
if (!ModulesCachePath.empty())
DepFS->setBypassedPathPrefix(ModulesCachePath);
ScanInstance.setDependencyDirectivesGetter(
std::make_unique<ScanningDependencyDirectivesGetter>(
ScanInstance.getFileManager()));
}
}
std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation(
const CompilerInvocation &Invocation,
const DependencyScanningService &Service) {
auto ScanInvocation = std::make_shared<CompilerInvocation>(Invocation);
sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts());
ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
true;
if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp =
Service.getOpts().BuildSessionTimestamp;
ScanInvocation->getFrontendOpts().DisableFree = false;
ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false;
ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false;
ScanInvocation->getFrontendOpts().GenReducedBMI = false;
ScanInvocation->getFrontendOpts().ModuleOutputPath.clear();
// This will prevent us compiling individual modules asynchronously since
// FileManager is not thread-safe, but it does improve performance for now.
ScanInvocation->getFrontendOpts().ModulesShareFileManager = true;
ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw";
ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage =
any(Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS);
// Consider different header search and diagnostic options to create
// different modules. This avoids the unsound aliasing of module PCMs.
//
// TODO: Implement diagnostic bucketing to reduce the impact of strict
// context hashing.
ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true;
ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
true;
ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
// Ensure that the scanner does not create new dependency collectors,
// and thus won't write out the extra '.d' files to disk.
ScanInvocation->getDependencyOutputOpts() = {};
return ScanInvocation;
}
llvm::SmallVector<StringRef>
dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) {
// Create a collection of stable directories derived from the ScanInstance
// for determining whether module dependencies would fully resolve from
// those directories.
llvm::SmallVector<StringRef> StableDirs;
const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot))
StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
return StableDirs;
}
std::optional<PrebuiltModulesAttrsMap>
dependencies::computePrebuiltModulesASTMap(
CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) {
// Store a mapping of prebuilt module files and their properties like header
// search options. This will prevent the implicit build to create duplicate
// modules and will force reuse of the existing prebuilt module files
// instead.
PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
if (visitPrebuiltModule(
ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs))
return {};
return PrebuiltModulesASTMap;
}
std::unique_ptr<DependencyOutputOptions>
dependencies::createDependencyOutputOptions(
const CompilerInvocation &Invocation) {
auto Opts = std::make_unique<DependencyOutputOptions>(
Invocation.getDependencyOutputOpts());
// We need at least one -MT equivalent for the generator of make dependency
// files to work.
if (Opts->Targets.empty())
Opts->Targets = {deduceDepTarget(Invocation.getFrontendOpts().OutputFile,
Invocation.getFrontendOpts().Inputs)};
Opts->IncludeSystemHeaders = true;
return Opts;
}
std::shared_ptr<ModuleDepCollector>
dependencies::initializeScanInstanceDependencyCollector(
CompilerInstance &ScanInstance,
std::unique_ptr<DependencyOutputOptions> DepOutputOpts,
StringRef WorkingDirectory, DependencyConsumer &Consumer,
DependencyScanningService &Service, CompilerInvocation &Inv,
DependencyActionController &Controller,
PrebuiltModulesAttrsMap PrebuiltModulesASTMap,
llvm::SmallVector<StringRef> &StableDirs) {
std::shared_ptr<ModuleDepCollector> MDC;
switch (Service.getOpts().Format) {
case ScanningOutputFormat::Make:
ScanInstance.addDependencyCollector(
std::make_shared<DependencyConsumerForwarder>(
std::move(DepOutputOpts), WorkingDirectory, Consumer));
break;
case ScanningOutputFormat::P1689:
case ScanningOutputFormat::Full:
MDC = std::make_shared<ModuleDepCollector>(
Service, std::move(DepOutputOpts), ScanInstance, Consumer, Controller,
Inv, std::move(PrebuiltModulesASTMap), StableDirs);
ScanInstance.addDependencyCollector(MDC);
break;
}
return MDC;
}
/// Manages (and terminates) the asynchronous compilation of modules.
class AsyncModuleCompiles {
std::mutex Mutex;
bool Stop = false;
// FIXME: Have the service own a thread pool and use that instead.
std::vector<std::thread> Compiles;
public:
/// Registers the module compilation, unless this instance is about to be
/// destroyed.
void add(llvm::unique_function<void()> Compile) {
std::lock_guard<std::mutex> Lock(Mutex);
if (!Stop)
Compiles.emplace_back(std::move(Compile));
}
~AsyncModuleCompiles() {
{
// Prevent registration of further module compiles.
std::lock_guard<std::mutex> Lock(Mutex);
Stop = true;
}
// Wait for outstanding module compiles to finish.
for (std::thread &Compile : Compiles)
Compile.join();
}
};
struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction {
DependencyScanningService &Service;
AsyncModuleCompiles &Compiles;
SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service,
AsyncModuleCompiles &Compiles)
: Service(Service), Compiles(Compiles) {}
bool BeginSourceFileAction(CompilerInstance &CI) override;
};
/// The preprocessor callback that takes care of initiating an asynchronous
/// module compilation if needed.
struct AsyncModuleCompile : PPCallbacks {
CompilerInstance &CI;
DependencyScanningService &Service;
AsyncModuleCompiles &Compiles;
AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service,
AsyncModuleCompiles &Compiles)
: CI(CI), Service(Service), Compiles(Compiles) {}
void moduleLoadSkipped(Module *M) override {
M = M->getTopLevelModule();
HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
ModuleCache &ModCache = CI.getModuleCache();
std::string ModuleFileName = HS.getCachedModuleFileName(M);
uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName);
// Someone else already built/validated the PCM.
if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp)
return;
if (!CI.getASTReader())
CI.createASTReader();
SmallVector<ASTReader::ImportedModule, 0> Imported;
// Only calling ReadASTCore() to avoid the expensive eager deserialization
// of the clang::Module objects in ReadAST().
// FIXME: Consider doing this in the new thread depending on how expensive
// the read turns out to be.
switch (CI.getASTReader()->ReadASTCore(
ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(),
nullptr, Imported, {}, {}, {},
ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing |
ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) {
case ASTReader::Success:
// We successfully read a valid, up-to-date PCM.
// FIXME: This could update the timestamp. Regular calls to
// ASTReader::ReadAST() would do so unless they encountered corrupted
// AST block, corrupted extension block, or did not read the expected
// top-level module.
return;
case ASTReader::OutOfDate:
case ASTReader::Missing:
// The most interesting case.
break;
default:
// Let the regular scan diagnose this.
return;
}
ModCache.prepareForGetLock(ModuleFileName);
auto Lock = ModCache.getLock(ModuleFileName);
bool Owned;
llvm::Error LockErr = Lock->tryLock().moveInto(Owned);
// Someone else is building the PCM right now.
if (!LockErr && !Owned)
return;
// We should build the PCM.
IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
Service.getSharedCache(), Service.getOpts().MakeVFS());
VFS = createVFSFromCompilerInvocation(CI.getInvocation(),
CI.getDiagnostics(), std::move(VFS));
auto DC = std::make_unique<DiagnosticConsumer>();
auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries());
CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC,
std::move(MC));
auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
CloneConfig);
auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName,
CloneConfig);
// Note: This lock belongs to a module cache that might not outlive the
// thread. This works, because the in-process lock only refers to an object
// managed by the service, which does outlive the thread.
Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1),
ModCI2 = std::move(ModCI2), DC = std::move(DC),
Service = &Service, Compiles = &Compiles] {
llvm::CrashRecoveryContext CRC;
(void)CRC.RunSafely([&] {
// Quickly discovers and compiles modules for the real scan below.
SingleModuleWithAsyncModuleCompiles Action1(*Service, *Compiles);
(void)ModCI1->ExecuteAction(Action1);
// The real scan below.
ModCI2->getPreprocessorOpts().SingleModuleParseMode = false;
GenerateModuleFromModuleMapAction Action2;
(void)ModCI2->ExecuteAction(Action2);
});
});
}
};
/// Runs the preprocessor on a TU with single-module-parse-mode and compiles
/// modules asynchronously without blocking or importing them.
struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction {
DependencyScanningService &Service;
AsyncModuleCompiles &Compiles;
SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service,
AsyncModuleCompiles &Compiles)
: Service(Service), Compiles(Compiles) {}
bool BeginSourceFileAction(CompilerInstance &CI) override {
CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
CI.getPreprocessor().addPPCallbacks(
std::make_unique<AsyncModuleCompile>(CI, Service, Compiles));
return true;
}
};
bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction(
CompilerInstance &CI) {
CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true;
CI.getPreprocessor().addPPCallbacks(
std::make_unique<AsyncModuleCompile>(CI, Service, Compiles));
return true;
}
bool DependencyScanningAction::runInvocation(
std::string Executable,
std::unique_ptr<CompilerInvocation> OriginalInvocation,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
DiagnosticConsumer *DiagConsumer) {
// Making sure that we canonicalize the defines early to avoid unnecessary
// variants in both the scanner and in the resulting explicit command lines.
if (any(Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros))
canonicalizeDefines(OriginalInvocation->getPreprocessorOpts());
if (Scanned) {
// Scanning runs once for the first -cc1 invocation in a chain of driver
// jobs. For any dependent jobs, reuse the scanning result and just
// update the new invocation.
// FIXME: to support multi-arch builds, each arch requires a separate scan
if (MDC)
MDC->applyDiscoveredDependencies(*OriginalInvocation);
Consumer.handleBuildCommand(
{Executable, OriginalInvocation->getCC1CommandLine()});
return true;
}
Scanned = true;
// Create a compiler instance to handle the actual work.
auto ScanInvocation =
createScanCompilerInvocation(*OriginalInvocation, Service);
// Quickly discovers and compiles modules for the real scan below.
std::optional<AsyncModuleCompiles> AsyncCompiles;
if (Service.getOpts().AsyncScanModules) {
auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
auto ScanInstanceStorage = std::make_unique<CompilerInstance>(
std::make_shared<CompilerInvocation>(*ScanInvocation), PCHContainerOps,
std::move(ModCache));
CompilerInstance &ScanInstance = *ScanInstanceStorage;
DiagnosticConsumer DiagConsumer;
initializeScanCompilerInstance(ScanInstance, FS, &DiagConsumer, Service,
DepFS);
// FIXME: Do this only once.
SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
auto MaybePrebuiltModulesASTMap =
computePrebuiltModulesASTMap(ScanInstance, StableDirs);
if (!MaybePrebuiltModulesASTMap)
return false;
// Normally this would be handled by GeneratePCHAction
if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH)
ScanInstance.getLangOpts().CompilingPCH = true;
AsyncCompiles.emplace();
SingleTUWithAsyncModuleCompiles Action(Service, *AsyncCompiles);
(void)ScanInstance.ExecuteAction(Action);
}
auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
ScanInstanceStorage.emplace(std::move(ScanInvocation),
std::move(PCHContainerOps), std::move(ModCache));
CompilerInstance &ScanInstance = *ScanInstanceStorage;
initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service,
DepFS);
llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance);
auto MaybePrebuiltModulesASTMap =
computePrebuiltModulesASTMap(ScanInstance, StableDirs);
if (!MaybePrebuiltModulesASTMap)
return false;
auto DepOutputOpts = createDependencyOutputOptions(*OriginalInvocation);
MDC = initializeScanInstanceDependencyCollector(
ScanInstance, std::move(DepOutputOpts), WorkingDirectory, Consumer,
Service, *OriginalInvocation, Controller, *MaybePrebuiltModulesASTMap,
StableDirs);
if (ScanInstance.getDiagnostics().hasErrorOccurred())
return false;
ReadPCHAndPreprocessAction Action;
const bool Result = ScanInstance.ExecuteAction(Action);
if (Result) {
if (MDC)
MDC->applyDiscoveredDependencies(*OriginalInvocation);
Consumer.handleBuildCommand(
{Executable, OriginalInvocation->getCC1CommandLine()});
}
return Result;
}
Reference in New Issue View Git Blame Copy Permalink