shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
Cyndy Ishida 1365b5b1ad
[clang][DependencyScanning] Track dependencies from prebuilt modules to determine IsInStableDir (#132237) 
When a module is being scanned, it can depend on modules that have
already been built from a pch dependency. When this happens, the pcm
files are reused for the module dependencies. When this is the case,
check if input files recorded from the PCMs come from the provided
stable directories transitively since the scanner will not have access
to the full set of file dependencies from prebuilt modules.
2025-04-08 15:48:25 -07:00

847 lines
35 KiB
C++
Raw Blame History

//===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Basic/DiagnosticDriver.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/DiagnosticSerialization.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/Job.h"
#include "clang/Driver/Tool.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/FrontendActions.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h"
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
#include "clang/Tooling/Tooling.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/TargetParser/Host.h"
#include <optional>
using namespace clang;
using namespace tooling;
using namespace dependencies;
namespace {
/// Forwards the gatherered dependencies to the consumer.
class DependencyConsumerForwarder : public DependencyFileGenerator {
public:
DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
StringRef WorkingDirectory, DependencyConsumer &C)
: DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
Opts(std::move(Opts)), C(C) {}
void finishedMainFile(DiagnosticsEngine &Diags) override {
C.handleDependencyOutputOpts(*Opts);
llvm::SmallString<256> CanonPath;
for (const auto &File : getDependencies()) {
CanonPath = File;
llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
llvm::sys::fs::make_absolute(WorkingDirectory, CanonPath);
C.handleFileDependency(CanonPath);
}
}
private:
StringRef WorkingDirectory;
std::unique_ptr<DependencyOutputOptions> Opts;
DependencyConsumer &C;
};
static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
const HeaderSearchOptions &ExistingHSOpts,
DiagnosticsEngine *Diags,
const LangOptions &LangOpts) {
if (LangOpts.Modules) {
if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
if (Diags) {
Diags->Report(diag::warn_pch_vfsoverlay_mismatch);
auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
if (VFSOverlays.empty()) {
Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;
} else {
std::string Files = llvm::join(VFSOverlays, "\n");
Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;
}
};
VFSNote(0, HSOpts.VFSOverlayFiles);
VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
}
}
}
return false;
}
using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
/// A listener that collects the imported modules and the input
/// files. While visiting, collect vfsoverlays and file inputs that determine
/// whether prebuilt modules fully resolve in stable directories.
class PrebuiltModuleListener : public ASTReaderListener {
public:
PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
llvm::SmallVector<std::string> &NewModuleFiles,
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
const HeaderSearchOptions &HSOpts,
const LangOptions &LangOpts, DiagnosticsEngine &Diags,
const llvm::SmallVector<StringRef> &StableDirs)
: PrebuiltModuleFiles(PrebuiltModuleFiles),
NewModuleFiles(NewModuleFiles),
PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
bool needsImportVisitation() const override { return true; }
bool needsInputFileVisitation() override { return true; }
bool needsSystemInputFileVisitation() override { return true; }
/// Accumulate the modules are transitively depended on by the initial
/// prebuilt module.
void visitImport(StringRef ModuleName, StringRef Filename) override {
if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
NewModuleFiles.push_back(Filename.str());
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
if (auto It = PrebuiltModulesASTMap.find(CurrentFile);
It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
PrebuiltModule.addDependent(It->getKey());
}
/// For each input file discovered, check whether it's external path is in a
/// stable directory. Traversal is stopped if the current module is not
/// considered stable.
bool visitInputFile(StringRef FilenameAsRequested, StringRef Filename,
bool isSystem, bool isOverridden,
bool isExplicitModule) override {
if (StableDirs.empty())
return false;
auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
(!PrebuiltEntryIt->second.isInStableDir()))
return false;
PrebuiltEntryIt->second.setInStableDir(
isPathInStableDir(StableDirs, Filename));
return PrebuiltEntryIt->second.isInStableDir();
}
/// Update which module that is being actively traversed.
void visitModuleFile(StringRef Filename,
serialization::ModuleKind Kind) override {
// If the CurrentFile is not
// considered stable, update any of it's transitive dependents.
auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
!PrebuiltEntryIt->second.isInStableDir())
PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
PrebuiltModulesASTMap);
CurrentFile = Filename;
}
/// Check the header search options for a given module when considering
/// if the module comes from stable directories.
bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
StringRef ModuleFilename,
StringRef SpecificModuleCachePath,
bool Complain) override {
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
if (PrebuiltModule.isInStableDir())
PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts));
return false;
}
/// Accumulate vfsoverlays used to build these prebuilt modules.
bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
bool Complain) override {
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
PrebuiltModule.setVFS(
llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
return checkHeaderSearchPaths(
HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
}
private:
PrebuiltModuleFilesT &PrebuiltModuleFiles;
llvm::SmallVector<std::string> &NewModuleFiles;
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
const HeaderSearchOptions &ExistingHSOpts;
const LangOptions &ExistingLangOpts;
DiagnosticsEngine &Diags;
std::string CurrentFile;
const llvm::SmallVector<StringRef> &StableDirs;
};
/// Visit the given prebuilt module and collect all of the modules it
/// transitively imports and contributing input files.
static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
CompilerInstance &CI,
PrebuiltModuleFilesT &ModuleFiles,
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
DiagnosticsEngine &Diags) {
// Gather the set of stable directories to use as transitive dependencies are
// discovered.
llvm::SmallVector<StringRef> StableDirs;
std::string SysrootToUse(CI.getHeaderSearchOpts().Sysroot);
if (!SysrootToUse.empty() &&
(llvm::sys::path::root_directory(SysrootToUse) != SysrootToUse))
StableDirs = {SysrootToUse, CI.getHeaderSearchOpts().ResourceDir};
// List of module files to be processed.
llvm::SmallVector<std::string> Worklist;
PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
CI.getHeaderSearchOpts(), CI.getLangOpts(),
Diags, StableDirs);
Listener.visitModuleFile(PrebuiltModuleFilename,
serialization::MK_ExplicitModule);
if (ASTReader::readASTFileControlBlock(
PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),
CI.getPCHContainerReader(),
/*FindModuleFileExtensions=*/false, Listener,
/*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))
return true;
while (!Worklist.empty()) {
Listener.visitModuleFile(Worklist.back(), serialization::MK_ExplicitModule);
if (ASTReader::readASTFileControlBlock(
Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
CI.getPCHContainerReader(),
/*FindModuleFileExtensions=*/false, Listener,
/*ValidateDiagnosticOptions=*/false))
return true;
}
return false;
}
/// Transform arbitrary file name into an object-like file name.
static std::string makeObjFileName(StringRef FileName) {
SmallString<128> ObjFileName(FileName);
llvm::sys::path::replace_extension(ObjFileName, "o");
return std::string(ObjFileName);
}
/// Deduce the dependency target based on the output file and input files.
static std::string
deduceDepTarget(const std::string &OutputFile,
const SmallVectorImpl<FrontendInputFile> &InputFiles) {
if (OutputFile != "-")
return OutputFile;
if (InputFiles.empty() || !InputFiles.front().isFile())
return "clang-scan-deps\\ dependency";
return makeObjFileName(InputFiles.front().getFile());
}
/// Sanitize diagnostic options for dependency scan.
static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
// Don't print 'X warnings and Y errors generated'.
DiagOpts.ShowCarets = false;
// Don't write out diagnostic file.
DiagOpts.DiagnosticSerializationFile.clear();
// Don't emit warnings except for scanning specific warnings.
// TODO: It would be useful to add a more principled way to ignore all
// warnings that come from source code. The issue is that we need to
// ignore warnings that could be surpressed by
// `#pragma clang diagnostic`, while still allowing some scanning
// warnings for things we're not ready to turn into errors yet.
// See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) {
return llvm::StringSwitch<bool>(Warning)
.Cases("pch-vfs-diff", "error=pch-vfs-diff", false)
.StartsWith("no-error=", false)
.Default(true);
});
}
// Clang implements -D and -U by splatting text into a predefines buffer. This
// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
// define the same macro, or adding C++ style comments before the macro name.
//
// This function checks that the first non-space characters in the macro
// obviously form an identifier that can be uniqued on without lexing. Failing
// to do this could lead to changing the final definition of a macro.
//
// We could set up a preprocessor and actually lex the name, but that's very
// heavyweight for a situation that will almost never happen in practice.
static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
StringRef Name = Macro.split("=").first.ltrim(" \t");
std::size_t I = 0;
auto FinishName = [&]() -> std::optional<StringRef> {
StringRef SimpleName = Name.slice(0, I);
if (SimpleName.empty())
return std::nullopt;
return SimpleName;
};
for (; I != Name.size(); ++I) {
switch (Name[I]) {
case '(': // Start of macro parameter list
case ' ': // End of macro name
case '\t':
return FinishName();
case '_':
continue;
default:
if (llvm::isAlnum(Name[I]))
continue;
return std::nullopt;
}
}
return FinishName();
}
static void canonicalizeDefines(PreprocessorOptions &PPOpts) {
using MacroOpt = std::pair<StringRef, std::size_t>;
std::vector<MacroOpt> SimpleNames;
SimpleNames.reserve(PPOpts.Macros.size());
std::size_t Index = 0;
for (const auto &M : PPOpts.Macros) {
auto SName = getSimpleMacroName(M.first);
// Skip optimizing if we can't guarantee we can preserve relative order.
if (!SName)
return;
SimpleNames.emplace_back(*SName, Index);
++Index;
}
llvm::stable_sort(SimpleNames, llvm::less_first());
// Keep the last instance of each macro name by going in reverse
auto NewEnd = std::unique(
SimpleNames.rbegin(), SimpleNames.rend(),
[](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
SimpleNames.erase(SimpleNames.begin(), NewEnd.base());
// Apply permutation.
decltype(PPOpts.Macros) NewMacros;
NewMacros.reserve(SimpleNames.size());
for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
std::size_t OriginalIndex = SimpleNames[I].second;
// We still emit undefines here as they may be undefining a predefined macro
NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex]));
}
std::swap(PPOpts.Macros, NewMacros);
}
/// A clang tool that runs the preprocessor in a mode that's optimized for
/// dependency scanning for the given compiler invocation.
class DependencyScanningAction : public tooling::ToolAction {
public:
DependencyScanningAction(
DependencyScanningService &Service, StringRef WorkingDirectory,
DependencyConsumer &Consumer, DependencyActionController &Controller,
llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
bool DisableFree, std::optional<StringRef> ModuleName = std::nullopt)
: Service(Service), WorkingDirectory(WorkingDirectory),
Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)),
DisableFree(DisableFree), ModuleName(ModuleName) {}
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
FileManager *DriverFileMgr,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
DiagnosticConsumer *DiagConsumer) override {
// Make a deep copy of the original Clang invocation.
CompilerInvocation OriginalInvocation(*Invocation);
// Restore the value of DisableFree, which may be modified by Tooling.
OriginalInvocation.getFrontendOpts().DisableFree = DisableFree;
if (any(Service.getOptimizeArgs() & ScanningOptimizations::Macros))
canonicalizeDefines(OriginalInvocation.getPreprocessorOpts());
if (Scanned) {
// Scanning runs once for the first -cc1 invocation in a chain of driver
// jobs. For any dependent jobs, reuse the scanning result and just
// update the LastCC1Arguments to correspond to the new invocation.
// FIXME: to support multi-arch builds, each arch requires a separate scan
setLastCC1Arguments(std::move(OriginalInvocation));
return true;
}
Scanned = true;
// Create a compiler instance to handle the actual work.
auto ModCache = makeInProcessModuleCache(Service.getModuleCacheMutexes());
ScanInstanceStorage.emplace(std::move(PCHContainerOps), ModCache.get());
CompilerInstance &ScanInstance = *ScanInstanceStorage;
ScanInstance.setInvocation(std::move(Invocation));
ScanInstance.setBuildingModule(false);
// Create the compiler's actual diagnostics engine.
sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
assert(!DiagConsumerFinished && "attempt to reuse finished consumer");
ScanInstance.createDiagnostics(DriverFileMgr->getVirtualFileSystem(),
DiagConsumer, /*ShouldOwnClient=*/false);
if (!ScanInstance.hasDiagnostics())
return false;
ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
true;
ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
// This will prevent us compiling individual modules asynchronously since
// FileManager is not thread-safe, but it does improve performance for now.
ScanInstance.getFrontendOpts().ModulesShareFileManager = true;
ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";
ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage =
any(Service.getOptimizeArgs() & ScanningOptimizations::VFS);
// Support for virtual file system overlays.
auto FS = createVFSFromCompilerInvocation(
ScanInstance.getInvocation(), ScanInstance.getDiagnostics(),
DriverFileMgr->getVirtualFileSystemPtr());
// Use the dependency scanning optimized file system if requested to do so.
if (DepFS) {
StringRef ModulesCachePath =
ScanInstance.getHeaderSearchOpts().ModuleCachePath;
DepFS->resetBypassedPathPrefix();
if (!ModulesCachePath.empty())
DepFS->setBypassedPathPrefix(ModulesCachePath);
ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
[LocalDepFS = DepFS](FileEntryRef File)
-> std::optional<ArrayRef<dependency_directives_scan::Directive>> {
if (llvm::ErrorOr<EntryRef> Entry =
LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry))
return Entry->getDirectiveTokens();
return std::nullopt;
};
}
// Create a new FileManager to match the invocation's FileSystemOptions.
auto *FileMgr = ScanInstance.createFileManager(FS);
ScanInstance.createSourceManager(*FileMgr);
// Store a mapping of prebuilt module files and their properties like header
// search options. This will prevent the implicit build to create duplicate
// modules and will force reuse of the existing prebuilt module files
// instead.
PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
if (visitPrebuiltModule(
ScanInstance.getPreprocessorOpts().ImplicitPCHInclude,
ScanInstance,
ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
PrebuiltModulesASTMap, ScanInstance.getDiagnostics()))
return false;
// Create the dependency collector that will collect the produced
// dependencies.
//
// This also moves the existing dependency output options from the
// invocation to the collector. The options in the invocation are reset,
// which ensures that the compiler won't create new dependency collectors,
// and thus won't write out the extra '.d' files to disk.
auto Opts = std::make_unique<DependencyOutputOptions>();
std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
// We need at least one -MT equivalent for the generator of make dependency
// files to work.
if (Opts->Targets.empty())
Opts->Targets = {
deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
ScanInstance.getFrontendOpts().Inputs)};
Opts->IncludeSystemHeaders = true;
switch (Service.getFormat()) {
case ScanningOutputFormat::Make:
ScanInstance.addDependencyCollector(
std::make_shared<DependencyConsumerForwarder>(
std::move(Opts), WorkingDirectory, Consumer));
break;
case ScanningOutputFormat::P1689:
case ScanningOutputFormat::Full:
MDC = std::make_shared<ModuleDepCollector>(
Service, std::move(Opts), ScanInstance, Consumer, Controller,
OriginalInvocation, std::move(PrebuiltModulesASTMap));
ScanInstance.addDependencyCollector(MDC);
break;
}
// Consider different header search and diagnostic options to create
// different modules. This avoids the unsound aliasing of module PCMs.
//
// TODO: Implement diagnostic bucketing to reduce the impact of strict
// context hashing.
ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
true;
// Avoid some checks and module map parsing when loading PCM files.
ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false;
std::unique_ptr<FrontendAction> Action;
if (Service.getFormat() == ScanningOutputFormat::P1689)
Action = std::make_unique<PreprocessOnlyAction>();
else if (ModuleName)
Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
else
Action = std::make_unique<ReadPCHAndPreprocessAction>();
if (ScanInstance.getDiagnostics().hasErrorOccurred())
return false;
const bool Result = ScanInstance.ExecuteAction(*Action);
// ExecuteAction is responsible for calling finish.
DiagConsumerFinished = true;
if (Result)
setLastCC1Arguments(std::move(OriginalInvocation));
// Propagate the statistics to the parent FileManager.
DriverFileMgr->AddStats(ScanInstance.getFileManager());
return Result;
}
bool hasScanned() const { return Scanned; }
bool hasDiagConsumerFinished() const { return DiagConsumerFinished; }
/// Take the cc1 arguments corresponding to the most recent invocation used
/// with this action. Any modifications implied by the discovered dependencies
/// will have already been applied.
std::vector<std::string> takeLastCC1Arguments() {
std::vector<std::string> Result;
std::swap(Result, LastCC1Arguments); // Reset LastCC1Arguments to empty.
return Result;
}
private:
void setLastCC1Arguments(CompilerInvocation &&CI) {
if (MDC)
MDC->applyDiscoveredDependencies(CI);
LastCC1Arguments = CI.getCC1CommandLine();
}
DependencyScanningService &Service;
StringRef WorkingDirectory;
DependencyConsumer &Consumer;
DependencyActionController &Controller;
llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
bool DisableFree;
std::optional<StringRef> ModuleName;
std::optional<CompilerInstance> ScanInstanceStorage;
std::shared_ptr<ModuleDepCollector> MDC;
std::vector<std::string> LastCC1Arguments;
bool Scanned = false;
bool DiagConsumerFinished = false;
};
} // end anonymous namespace
DependencyScanningWorker::DependencyScanningWorker(
DependencyScanningService &Service,
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
: Service(Service) {
PCHContainerOps = std::make_shared<PCHContainerOperations>();
// We need to read object files from PCH built outside the scanner.
PCHContainerOps->registerReader(
std::make_unique<ObjectFilePCHContainerReader>());
// The scanner itself writes only raw ast files.
PCHContainerOps->registerWriter(std::make_unique<RawPCHContainerWriter>());
if (Service.shouldTraceVFS())
FS = llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(std::move(FS));
switch (Service.getMode()) {
case ScanningMode::DependencyDirectivesScan:
DepFS =
new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
BaseFS = DepFS;
break;
case ScanningMode::CanonicalPreprocessing:
DepFS = nullptr;
BaseFS = FS;
break;
}
}
static std::unique_ptr<DiagnosticOptions>
createDiagOptions(const std::vector<std::string> &CommandLine) {
std::vector<const char *> CLI;
for (const std::string &Arg : CommandLine)
CLI.push_back(Arg.c_str());
auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
sanitizeDiagOpts(*DiagOpts);
return DiagOpts;
}
llvm::Error DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
std::optional<llvm::MemoryBufferRef> TUBuffer) {
// Capture the emitted diagnostics and report them to the client
// in the case of a failure.
std::string DiagnosticOutput;
llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
auto DiagOpts = createDiagOptions(CommandLine);
TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release());
if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
DiagPrinter, TUBuffer))
return llvm::Error::success();
return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
llvm::inconvertibleErrorCode());
}
llvm::Error DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
StringRef ModuleName) {
// Capture the emitted diagnostics and report them to the client
// in the case of a failure.
std::string DiagnosticOutput;
llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
auto DiagOpts = createDiagOptions(CommandLine);
TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release());
if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
DiagPrinter, ModuleName))
return llvm::Error::success();
return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
llvm::inconvertibleErrorCode());
}
static bool forEachDriverJob(
ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, FileManager &FM,
llvm::function_ref<bool(const driver::Command &Cmd)> Callback) {
SmallVector<const char *, 256> Argv;
Argv.reserve(ArgStrs.size());
for (const std::string &Arg : ArgStrs)
Argv.push_back(Arg.c_str());
llvm::vfs::FileSystem *FS = &FM.getVirtualFileSystem();
std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
Argv[0], llvm::sys::getDefaultTargetTriple(), Diags,
"clang LLVM compiler", FS);
Driver->setTitle("clang_based_tool");
llvm::BumpPtrAllocator Alloc;
bool CLMode = driver::IsClangCL(
driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1)));
if (llvm::Error E = driver::expandResponseFiles(Argv, CLMode, Alloc, FS)) {
Diags.Report(diag::err_drv_expand_response_file)
<< llvm::toString(std::move(E));
return false;
}
const std::unique_ptr<driver::Compilation> Compilation(
Driver->BuildCompilation(llvm::ArrayRef(Argv)));
if (!Compilation)
return false;
if (Compilation->containsError())
return false;
for (const driver::Command &Job : Compilation->getJobs()) {
if (!Callback(Job))
return false;
}
return true;
}
static bool createAndRunToolInvocation(
std::vector<std::string> CommandLine, DependencyScanningAction &Action,
FileManager &FM,
std::shared_ptr<clang::PCHContainerOperations> &PCHContainerOps,
DiagnosticsEngine &Diags, DependencyConsumer &Consumer) {
// Save executable path before providing CommandLine to ToolInvocation
std::string Executable = CommandLine[0];
ToolInvocation Invocation(std::move(CommandLine), &Action, &FM,
PCHContainerOps);
Invocation.setDiagnosticConsumer(Diags.getClient());
Invocation.setDiagnosticOptions(&Diags.getDiagnosticOptions());
if (!Invocation.run())
return false;
std::vector<std::string> Args = Action.takeLastCC1Arguments();
Consumer.handleBuildCommand({std::move(Executable), std::move(Args)});
return true;
}
bool DependencyScanningWorker::scanDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::optional<StringRef> ModuleName) {
auto FileMgr =
llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions{}, FS);
std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
llvm::transform(CommandLine, CCommandLine.begin(),
[](const std::string &Str) { return Str.c_str(); });
auto DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);
sanitizeDiagOpts(*DiagOpts);
IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
CompilerInstance::createDiagnostics(FileMgr->getVirtualFileSystem(),
DiagOpts.release(), &DC,
/*ShouldOwnClient=*/false);
// Although `Diagnostics` are used only for command-line parsing, the
// custom `DiagConsumer` might expect a `SourceManager` to be present.
SourceManager SrcMgr(*Diags, *FileMgr);
Diags->setSourceManager(&SrcMgr);
// DisableFree is modified by Tooling for running
// in-process; preserve the original value, which is
// always true for a driver invocation.
bool DisableFree = true;
DependencyScanningAction Action(Service, WorkingDirectory, Consumer,
Controller, DepFS, DisableFree, ModuleName);
bool Success = false;
if (CommandLine[1] == "-cc1") {
Success = createAndRunToolInvocation(CommandLine, Action, *FileMgr,
PCHContainerOps, *Diags, Consumer);
} else {
Success = forEachDriverJob(
CommandLine, *Diags, *FileMgr, [&](const driver::Command &Cmd) {
if (StringRef(Cmd.getCreator().getName()) != "clang") {
// Non-clang command. Just pass through to the dependency
// consumer.
Consumer.handleBuildCommand(
{Cmd.getExecutable(),
{Cmd.getArguments().begin(), Cmd.getArguments().end()}});
return true;
}
// Insert -cc1 comand line options into Argv
std::vector<std::string> Argv;
Argv.push_back(Cmd.getExecutable());
Argv.insert(Argv.end(), Cmd.getArguments().begin(),
Cmd.getArguments().end());
// Create an invocation that uses the underlying file
// system to ensure that any file system requests that
// are made by the driver do not go through the
// dependency scanning filesystem.
return createAndRunToolInvocation(std::move(Argv), Action, *FileMgr,
PCHContainerOps, *Diags, Consumer);
});
}
if (Success && !Action.hasScanned())
Diags->Report(diag::err_fe_expected_compiler_job)
<< llvm::join(CommandLine, " ");
// Ensure finish() is called even if we never reached ExecuteAction().
if (!Action.hasDiagConsumerFinished())
DC.finish();
return Success && Action.hasScanned();
}
bool DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, std::optional<llvm::MemoryBufferRef> TUBuffer) {
// Reset what might have been modified in the previous worker invocation.
BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
std::optional<std::vector<std::string>> ModifiedCommandLine;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;
// If we're scanning based on a module name alone, we don't expect the client
// to provide us with an input file. However, the driver really wants to have
// one. Let's just make it up to make the driver happy.
if (TUBuffer) {
auto OverlayFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
auto InMemoryFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
auto InputPath = TUBuffer->getBufferIdentifier();
InMemoryFS->addFile(
InputPath, 0,
llvm::MemoryBuffer::getMemBufferCopy(TUBuffer->getBuffer()));
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay =
InMemoryFS;
OverlayFS->pushOverlay(InMemoryOverlay);
ModifiedFS = OverlayFS;
ModifiedCommandLine = CommandLine;
ModifiedCommandLine->emplace_back(InputPath);
}
const std::vector<std::string> &FinalCommandLine =
ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;
return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer,
Controller, DC, FinalFS, /*ModuleName=*/std::nullopt);
}
bool DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, StringRef ModuleName) {
// Reset what might have been modified in the previous worker invocation.
BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
// If we're scanning based on a module name alone, we don't expect the client
// to provide us with an input file. However, the driver really wants to have
// one. Let's just make it up to make the driver happy.
auto OverlayFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
SmallString<128> FakeInputPath;
// TODO: We should retry the creation if the path already exists.
llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath,
/*MakeAbsolute=*/false);
InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
OverlayFS->pushOverlay(InMemoryOverlay);
auto ModifiedCommandLine = CommandLine;
ModifiedCommandLine.emplace_back(FakeInputPath);
return scanDependencies(WorkingDirectory, ModifiedCommandLine, Consumer,
Controller, DC, OverlayFS, ModuleName);
}
DependencyActionController::~DependencyActionController() {}
Reference in New Issue View Git Blame Copy Permalink