shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
James Y Knight 9ddbb478ce
NFC: Clean up construction of IntrusiveRefCntPtr from raw pointers for llvm::vfs::FileSystem. (#151407) 
This switches to `makeIntrusiveRefCnt<FileSystem>` where creating a new
object, and to passing/returning by `IntrusiveRefCntPtr<FileSystem>`
instead of `FileSystem*` or `FileSystem&`, when dealing with existing
objects.

Part of cleanup #151026.
2025-07-31 09:57:13 -04:00

860 lines
35 KiB
C++
Raw Blame History

//===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
#include "clang/Basic/DiagnosticDriver.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/DiagnosticSerialization.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/Job.h"
#include "clang/Driver/Tool.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/FrontendActions.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "clang/Serialization/ObjectFilePCHContainerReader.h"
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h"
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/TargetParser/Host.h"
#include <optional>
using namespace clang;
using namespace tooling;
using namespace dependencies;
namespace {
/// Forwards the gatherered dependencies to the consumer.
class DependencyConsumerForwarder : public DependencyFileGenerator {
public:
DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
StringRef WorkingDirectory, DependencyConsumer &C)
: DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory),
Opts(std::move(Opts)), C(C) {}
void finishedMainFile(DiagnosticsEngine &Diags) override {
C.handleDependencyOutputOpts(*Opts);
llvm::SmallString<256> CanonPath;
for (const auto &File : getDependencies()) {
CanonPath = File;
llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
llvm::sys::fs::make_absolute(WorkingDirectory, CanonPath);
C.handleFileDependency(CanonPath);
}
}
private:
StringRef WorkingDirectory;
std::unique_ptr<DependencyOutputOptions> Opts;
DependencyConsumer &C;
};
static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
const HeaderSearchOptions &ExistingHSOpts,
DiagnosticsEngine *Diags,
const LangOptions &LangOpts) {
if (LangOpts.Modules) {
if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) {
if (Diags) {
Diags->Report(diag::warn_pch_vfsoverlay_mismatch);
auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) {
if (VFSOverlays.empty()) {
Diags->Report(diag::note_pch_vfsoverlay_empty) << Type;
} else {
std::string Files = llvm::join(VFSOverlays, "\n");
Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files;
}
};
VFSNote(0, HSOpts.VFSOverlayFiles);
VFSNote(1, ExistingHSOpts.VFSOverlayFiles);
}
}
}
return false;
}
using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
/// A listener that collects the imported modules and the input
/// files. While visiting, collect vfsoverlays and file inputs that determine
/// whether prebuilt modules fully resolve in stable directories.
class PrebuiltModuleListener : public ASTReaderListener {
public:
PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
llvm::SmallVector<std::string> &NewModuleFiles,
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
const HeaderSearchOptions &HSOpts,
const LangOptions &LangOpts, DiagnosticsEngine &Diags,
const ArrayRef<StringRef> StableDirs)
: PrebuiltModuleFiles(PrebuiltModuleFiles),
NewModuleFiles(NewModuleFiles),
PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts),
ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {}
bool needsImportVisitation() const override { return true; }
bool needsInputFileVisitation() override { return true; }
bool needsSystemInputFileVisitation() override { return true; }
/// Accumulate the modules are transitively depended on by the initial
/// prebuilt module.
void visitImport(StringRef ModuleName, StringRef Filename) override {
if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
NewModuleFiles.push_back(Filename.str());
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
if (auto It = PrebuiltModulesASTMap.find(CurrentFile);
It != PrebuiltModulesASTMap.end() && CurrentFile != Filename)
PrebuiltModule.addDependent(It->getKey());
}
/// For each input file discovered, check whether it's external path is in a
/// stable directory. Traversal is stopped if the current module is not
/// considered stable.
bool visitInputFile(StringRef FilenameAsRequested, StringRef Filename,
bool isSystem, bool isOverridden,
bool isExplicitModule) override {
if (StableDirs.empty())
return false;
auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) ||
(!PrebuiltEntryIt->second.isInStableDir()))
return false;
PrebuiltEntryIt->second.setInStableDir(
isPathInStableDir(StableDirs, Filename));
return PrebuiltEntryIt->second.isInStableDir();
}
/// Update which module that is being actively traversed.
void visitModuleFile(StringRef Filename,
serialization::ModuleKind Kind) override {
// If the CurrentFile is not
// considered stable, update any of it's transitive dependents.
auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile);
if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) &&
!PrebuiltEntryIt->second.isInStableDir())
PrebuiltEntryIt->second.updateDependentsNotInStableDirs(
PrebuiltModulesASTMap);
CurrentFile = Filename;
}
/// Check the header search options for a given module when considering
/// if the module comes from stable directories.
bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
StringRef ModuleFilename,
StringRef SpecificModuleCachePath,
bool Complain) override {
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
if (PrebuiltModule.isInStableDir())
PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts));
return false;
}
/// Accumulate vfsoverlays used to build these prebuilt modules.
bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts,
bool Complain) override {
auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile);
PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second;
if (PrebuiltMapEntry.second)
PrebuiltModule.setInStableDir(!StableDirs.empty());
PrebuiltModule.setVFS(
llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles));
return checkHeaderSearchPaths(
HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts);
}
private:
PrebuiltModuleFilesT &PrebuiltModuleFiles;
llvm::SmallVector<std::string> &NewModuleFiles;
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap;
const HeaderSearchOptions &ExistingHSOpts;
const LangOptions &ExistingLangOpts;
DiagnosticsEngine &Diags;
std::string CurrentFile;
const ArrayRef<StringRef> StableDirs;
};
/// Visit the given prebuilt module and collect all of the modules it
/// transitively imports and contributing input files.
static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename,
CompilerInstance &CI,
PrebuiltModuleFilesT &ModuleFiles,
PrebuiltModulesAttrsMap &PrebuiltModulesASTMap,
DiagnosticsEngine &Diags,
const ArrayRef<StringRef> StableDirs) {
// List of module files to be processed.
llvm::SmallVector<std::string> Worklist;
PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap,
CI.getHeaderSearchOpts(), CI.getLangOpts(),
Diags, StableDirs);
Listener.visitModuleFile(PrebuiltModuleFilename,
serialization::MK_ExplicitModule);
if (ASTReader::readASTFileControlBlock(
PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(),
CI.getPCHContainerReader(),
/*FindModuleFileExtensions=*/false, Listener,
/*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate))
return true;
while (!Worklist.empty()) {
Listener.visitModuleFile(Worklist.back(), serialization::MK_ExplicitModule);
if (ASTReader::readASTFileControlBlock(
Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(),
CI.getPCHContainerReader(),
/*FindModuleFileExtensions=*/false, Listener,
/*ValidateDiagnosticOptions=*/false))
return true;
}
return false;
}
/// Transform arbitrary file name into an object-like file name.
static std::string makeObjFileName(StringRef FileName) {
SmallString<128> ObjFileName(FileName);
llvm::sys::path::replace_extension(ObjFileName, "o");
return std::string(ObjFileName);
}
/// Deduce the dependency target based on the output file and input files.
static std::string
deduceDepTarget(const std::string &OutputFile,
const SmallVectorImpl<FrontendInputFile> &InputFiles) {
if (OutputFile != "-")
return OutputFile;
if (InputFiles.empty() || !InputFiles.front().isFile())
return "clang-scan-deps\\ dependency";
return makeObjFileName(InputFiles.front().getFile());
}
/// Sanitize diagnostic options for dependency scan.
static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
// Don't print 'X warnings and Y errors generated'.
DiagOpts.ShowCarets = false;
// Don't write out diagnostic file.
DiagOpts.DiagnosticSerializationFile.clear();
// Don't emit warnings except for scanning specific warnings.
// TODO: It would be useful to add a more principled way to ignore all
// warnings that come from source code. The issue is that we need to
// ignore warnings that could be surpressed by
// `#pragma clang diagnostic`, while still allowing some scanning
// warnings for things we're not ready to turn into errors yet.
// See `test/ClangScanDeps/diagnostic-pragmas.c` for an example.
llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) {
return llvm::StringSwitch<bool>(Warning)
.Cases("pch-vfs-diff", "error=pch-vfs-diff", false)
.StartsWith("no-error=", false)
.Default(true);
});
}
// Clang implements -D and -U by splatting text into a predefines buffer. This
// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and
// define the same macro, or adding C++ style comments before the macro name.
//
// This function checks that the first non-space characters in the macro
// obviously form an identifier that can be uniqued on without lexing. Failing
// to do this could lead to changing the final definition of a macro.
//
// We could set up a preprocessor and actually lex the name, but that's very
// heavyweight for a situation that will almost never happen in practice.
static std::optional<StringRef> getSimpleMacroName(StringRef Macro) {
StringRef Name = Macro.split("=").first.ltrim(" \t");
std::size_t I = 0;
auto FinishName = [&]() -> std::optional<StringRef> {
StringRef SimpleName = Name.slice(0, I);
if (SimpleName.empty())
return std::nullopt;
return SimpleName;
};
for (; I != Name.size(); ++I) {
switch (Name[I]) {
case '(': // Start of macro parameter list
case ' ': // End of macro name
case '\t':
return FinishName();
case '_':
continue;
default:
if (llvm::isAlnum(Name[I]))
continue;
return std::nullopt;
}
}
return FinishName();
}
static void canonicalizeDefines(PreprocessorOptions &PPOpts) {
using MacroOpt = std::pair<StringRef, std::size_t>;
std::vector<MacroOpt> SimpleNames;
SimpleNames.reserve(PPOpts.Macros.size());
std::size_t Index = 0;
for (const auto &M : PPOpts.Macros) {
auto SName = getSimpleMacroName(M.first);
// Skip optimizing if we can't guarantee we can preserve relative order.
if (!SName)
return;
SimpleNames.emplace_back(*SName, Index);
++Index;
}
llvm::stable_sort(SimpleNames, llvm::less_first());
// Keep the last instance of each macro name by going in reverse
auto NewEnd = std::unique(
SimpleNames.rbegin(), SimpleNames.rend(),
[](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; });
SimpleNames.erase(SimpleNames.begin(), NewEnd.base());
// Apply permutation.
decltype(PPOpts.Macros) NewMacros;
NewMacros.reserve(SimpleNames.size());
for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) {
std::size_t OriginalIndex = SimpleNames[I].second;
// We still emit undefines here as they may be undefining a predefined macro
NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex]));
}
std::swap(PPOpts.Macros, NewMacros);
}
class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter {
DependencyScanningWorkerFilesystem *DepFS;
public:
ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) {
FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) {
auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS);
if (DFS) {
assert(!DepFS && "Found multiple scanning VFSs");
DepFS = DFS;
}
});
assert(DepFS && "Did not find scanning VFS");
}
std::unique_ptr<DependencyDirectivesGetter>
cloneFor(FileManager &FileMgr) override {
return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr);
}
std::optional<ArrayRef<dependency_directives_scan::Directive>>
operator()(FileEntryRef File) override {
return DepFS->getDirectiveTokens(File.getName());
}
};
/// A clang tool that runs the preprocessor in a mode that's optimized for
/// dependency scanning for the given compiler invocation.
class DependencyScanningAction {
public:
DependencyScanningAction(
DependencyScanningService &Service, StringRef WorkingDirectory,
DependencyConsumer &Consumer, DependencyActionController &Controller,
llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
std::optional<StringRef> ModuleName = std::nullopt)
: Service(Service), WorkingDirectory(WorkingDirectory),
Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)),
ModuleName(ModuleName) {}
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
DiagnosticConsumer *DiagConsumer) {
// Make a deep copy of the original Clang invocation.
CompilerInvocation OriginalInvocation(*Invocation);
if (any(Service.getOptimizeArgs() & ScanningOptimizations::Macros))
canonicalizeDefines(OriginalInvocation.getPreprocessorOpts());
if (Scanned) {
// Scanning runs once for the first -cc1 invocation in a chain of driver
// jobs. For any dependent jobs, reuse the scanning result and just
// update the LastCC1Arguments to correspond to the new invocation.
// FIXME: to support multi-arch builds, each arch requires a separate scan
setLastCC1Arguments(std::move(OriginalInvocation));
return true;
}
Scanned = true;
// Create a compiler instance to handle the actual work.
auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries());
ScanInstanceStorage.emplace(std::move(Invocation),
std::move(PCHContainerOps), ModCache.get());
CompilerInstance &ScanInstance = *ScanInstanceStorage;
ScanInstance.setBuildingModule(false);
// Create the compiler's actual diagnostics engine.
sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
assert(!DiagConsumerFinished && "attempt to reuse finished consumer");
ScanInstance.createDiagnostics(*FS, DiagConsumer,
/*ShouldOwnClient=*/false);
if (!ScanInstance.hasDiagnostics())
return false;
ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
true;
if (ScanInstance.getHeaderSearchOpts().ModulesValidateOncePerBuildSession)
ScanInstance.getHeaderSearchOpts().BuildSessionTimestamp =
Service.getBuildSessionTimestamp();
ScanInstance.getFrontendOpts().DisableFree = false;
ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
// This will prevent us compiling individual modules asynchronously since
// FileManager is not thread-safe, but it does improve performance for now.
ScanInstance.getFrontendOpts().ModulesShareFileManager = true;
ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";
ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage =
any(Service.getOptimizeArgs() & ScanningOptimizations::VFS);
// Support for virtual file system overlays.
FS = createVFSFromCompilerInvocation(ScanInstance.getInvocation(),
ScanInstance.getDiagnostics(),
std::move(FS));
// Create a new FileManager to match the invocation's FileSystemOptions.
auto *FileMgr = ScanInstance.createFileManager(FS);
// Use the dependency scanning optimized file system if requested to do so.
if (DepFS) {
StringRef ModulesCachePath =
ScanInstance.getHeaderSearchOpts().ModuleCachePath;
DepFS->resetBypassedPathPrefix();
if (!ModulesCachePath.empty())
DepFS->setBypassedPathPrefix(ModulesCachePath);
ScanInstance.setDependencyDirectivesGetter(
std::make_unique<ScanningDependencyDirectivesGetter>(*FileMgr));
}
ScanInstance.createSourceManager(*FileMgr);
// Create a collection of stable directories derived from the ScanInstance
// for determining whether module dependencies would fully resolve from
// those directories.
llvm::SmallVector<StringRef> StableDirs;
const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot;
if (!Sysroot.empty() &&
(llvm::sys::path::root_directory(Sysroot) != Sysroot))
StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
// Store a mapping of prebuilt module files and their properties like header
// search options. This will prevent the implicit build to create duplicate
// modules and will force reuse of the existing prebuilt module files
// instead.
PrebuiltModulesAttrsMap PrebuiltModulesASTMap;
if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
if (visitPrebuiltModule(
ScanInstance.getPreprocessorOpts().ImplicitPCHInclude,
ScanInstance,
ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs))
return false;
// Create the dependency collector that will collect the produced
// dependencies.
//
// This also moves the existing dependency output options from the
// invocation to the collector. The options in the invocation are reset,
// which ensures that the compiler won't create new dependency collectors,
// and thus won't write out the extra '.d' files to disk.
auto Opts = std::make_unique<DependencyOutputOptions>();
std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
// We need at least one -MT equivalent for the generator of make dependency
// files to work.
if (Opts->Targets.empty())
Opts->Targets = {
deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
ScanInstance.getFrontendOpts().Inputs)};
Opts->IncludeSystemHeaders = true;
switch (Service.getFormat()) {
case ScanningOutputFormat::Make:
ScanInstance.addDependencyCollector(
std::make_shared<DependencyConsumerForwarder>(
std::move(Opts), WorkingDirectory, Consumer));
break;
case ScanningOutputFormat::P1689:
case ScanningOutputFormat::Full:
MDC = std::make_shared<ModuleDepCollector>(
Service, std::move(Opts), ScanInstance, Consumer, Controller,
OriginalInvocation, std::move(PrebuiltModulesASTMap), StableDirs);
ScanInstance.addDependencyCollector(MDC);
break;
}
// Consider different header search and diagnostic options to create
// different modules. This avoids the unsound aliasing of module PCMs.
//
// TODO: Implement diagnostic bucketing to reduce the impact of strict
// context hashing.
ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true;
ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true;
ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true;
ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings =
true;
ScanInstance.getHeaderSearchOpts().ModulesForceValidateUserHeaders = false;
// Avoid some checks and module map parsing when loading PCM files.
ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false;
std::unique_ptr<FrontendAction> Action;
if (Service.getFormat() == ScanningOutputFormat::P1689)
Action = std::make_unique<PreprocessOnlyAction>();
else if (ModuleName)
Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
else
Action = std::make_unique<ReadPCHAndPreprocessAction>();
if (ScanInstance.getDiagnostics().hasErrorOccurred())
return false;
const bool Result = ScanInstance.ExecuteAction(*Action);
// ExecuteAction is responsible for calling finish.
DiagConsumerFinished = true;
if (Result)
setLastCC1Arguments(std::move(OriginalInvocation));
return Result;
}
bool hasScanned() const { return Scanned; }
bool hasDiagConsumerFinished() const { return DiagConsumerFinished; }
/// Take the cc1 arguments corresponding to the most recent invocation used
/// with this action. Any modifications implied by the discovered dependencies
/// will have already been applied.
std::vector<std::string> takeLastCC1Arguments() {
std::vector<std::string> Result;
std::swap(Result, LastCC1Arguments); // Reset LastCC1Arguments to empty.
return Result;
}
private:
void setLastCC1Arguments(CompilerInvocation &&CI) {
if (MDC)
MDC->applyDiscoveredDependencies(CI);
LastCC1Arguments = CI.getCC1CommandLine();
}
DependencyScanningService &Service;
StringRef WorkingDirectory;
DependencyConsumer &Consumer;
DependencyActionController &Controller;
llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
std::optional<StringRef> ModuleName;
std::optional<CompilerInstance> ScanInstanceStorage;
std::shared_ptr<ModuleDepCollector> MDC;
std::vector<std::string> LastCC1Arguments;
bool Scanned = false;
bool DiagConsumerFinished = false;
};
} // end anonymous namespace
DependencyScanningWorker::DependencyScanningWorker(
DependencyScanningService &Service,
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
: Service(Service) {
PCHContainerOps = std::make_shared<PCHContainerOperations>();
// We need to read object files from PCH built outside the scanner.
PCHContainerOps->registerReader(
std::make_unique<ObjectFilePCHContainerReader>());
// The scanner itself writes only raw ast files.
PCHContainerOps->registerWriter(std::make_unique<RawPCHContainerWriter>());
if (Service.shouldTraceVFS())
FS = llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(std::move(FS));
switch (Service.getMode()) {
case ScanningMode::DependencyDirectivesScan:
DepFS = llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
Service.getSharedCache(), FS);
BaseFS = DepFS;
break;
case ScanningMode::CanonicalPreprocessing:
DepFS = nullptr;
BaseFS = FS;
break;
}
}
static std::unique_ptr<DiagnosticOptions>
createDiagOptions(const std::vector<std::string> &CommandLine) {
std::vector<const char *> CLI;
for (const std::string &Arg : CommandLine)
CLI.push_back(Arg.c_str());
auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
sanitizeDiagOpts(*DiagOpts);
return DiagOpts;
}
llvm::Error DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
std::optional<llvm::MemoryBufferRef> TUBuffer) {
// Capture the emitted diagnostics and report them to the client
// in the case of a failure.
std::string DiagnosticOutput;
llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
auto DiagOpts = createDiagOptions(CommandLine);
TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, *DiagOpts);
if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
DiagPrinter, TUBuffer))
return llvm::Error::success();
return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
llvm::inconvertibleErrorCode());
}
llvm::Error DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
StringRef ModuleName) {
// Capture the emitted diagnostics and report them to the client
// in the case of a failure.
std::string DiagnosticOutput;
llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
auto DiagOpts = createDiagOptions(CommandLine);
TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, *DiagOpts);
if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
DiagPrinter, ModuleName))
return llvm::Error::success();
return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
llvm::inconvertibleErrorCode());
}
static bool forEachDriverJob(
ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
llvm::function_ref<bool(const driver::Command &Cmd)> Callback) {
SmallVector<const char *, 256> Argv;
Argv.reserve(ArgStrs.size());
for (const std::string &Arg : ArgStrs)
Argv.push_back(Arg.c_str());
std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>(
Argv[0], llvm::sys::getDefaultTargetTriple(), Diags,
"clang LLVM compiler", FS);
Driver->setTitle("clang_based_tool");
llvm::BumpPtrAllocator Alloc;
bool CLMode = driver::IsClangCL(
driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1)));
if (llvm::Error E =
driver::expandResponseFiles(Argv, CLMode, Alloc, FS.get())) {
Diags.Report(diag::err_drv_expand_response_file)
<< llvm::toString(std::move(E));
return false;
}
const std::unique_ptr<driver::Compilation> Compilation(
Driver->BuildCompilation(llvm::ArrayRef(Argv)));
if (!Compilation)
return false;
if (Compilation->containsError())
return false;
for (const driver::Command &Job : Compilation->getJobs()) {
if (!Callback(Job))
return false;
}
return true;
}
static bool createAndRunToolInvocation(
std::vector<std::string> CommandLine, DependencyScanningAction &Action,
IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::shared_ptr<clang::PCHContainerOperations> &PCHContainerOps,
DiagnosticsEngine &Diags, DependencyConsumer &Consumer) {
// Save executable path before providing CommandLine to ToolInvocation
std::string Executable = CommandLine[0];
llvm::opt::ArgStringList Argv;
for (const std::string &Str : ArrayRef(CommandLine).drop_front())
Argv.push_back(Str.c_str());
auto Invocation = std::make_shared<CompilerInvocation>();
if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) {
// FIXME: Should we just go on like cc1_main does?
return false;
}
if (!Action.runInvocation(std::move(Invocation), std::move(FS),
PCHContainerOps, Diags.getClient()))
return false;
std::vector<std::string> Args = Action.takeLastCC1Arguments();
Consumer.handleBuildCommand({std::move(Executable), std::move(Args)});
return true;
}
bool DependencyScanningWorker::scanDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::optional<StringRef> ModuleName) {
std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
llvm::transform(CommandLine, CCommandLine.begin(),
[](const std::string &Str) { return Str.c_str(); });
auto DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);
sanitizeDiagOpts(*DiagOpts);
auto Diags = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC,
/*ShouldOwnClient=*/false);
DependencyScanningAction Action(Service, WorkingDirectory, Consumer,
Controller, DepFS, ModuleName);
bool Success = false;
if (CommandLine[1] == "-cc1") {
Success = createAndRunToolInvocation(CommandLine, Action, FS,
PCHContainerOps, *Diags, Consumer);
} else {
Success = forEachDriverJob(
CommandLine, *Diags, FS, [&](const driver::Command &Cmd) {
if (StringRef(Cmd.getCreator().getName()) != "clang") {
// Non-clang command. Just pass through to the dependency
// consumer.
Consumer.handleBuildCommand(
{Cmd.getExecutable(),
{Cmd.getArguments().begin(), Cmd.getArguments().end()}});
return true;
}
// Insert -cc1 comand line options into Argv
std::vector<std::string> Argv;
Argv.push_back(Cmd.getExecutable());
llvm::append_range(Argv, Cmd.getArguments());
// Create an invocation that uses the underlying file
// system to ensure that any file system requests that
// are made by the driver do not go through the
// dependency scanning filesystem.
return createAndRunToolInvocation(std::move(Argv), Action, FS,
PCHContainerOps, *Diags, Consumer);
});
}
if (Success && !Action.hasScanned())
Diags->Report(diag::err_fe_expected_compiler_job)
<< llvm::join(CommandLine, " ");
// Ensure finish() is called even if we never reached ExecuteAction().
if (!Action.hasDiagConsumerFinished())
DC.finish();
return Success && Action.hasScanned();
}
bool DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, std::optional<llvm::MemoryBufferRef> TUBuffer) {
// Reset what might have been modified in the previous worker invocation.
BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
std::optional<std::vector<std::string>> ModifiedCommandLine;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;
// If we're scanning based on a module name alone, we don't expect the client
// to provide us with an input file. However, the driver really wants to have
// one. Let's just make it up to make the driver happy.
if (TUBuffer) {
auto OverlayFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
auto InMemoryFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
auto InputPath = TUBuffer->getBufferIdentifier();
InMemoryFS->addFile(
InputPath, 0,
llvm::MemoryBuffer::getMemBufferCopy(TUBuffer->getBuffer()));
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay =
InMemoryFS;
OverlayFS->pushOverlay(InMemoryOverlay);
ModifiedFS = OverlayFS;
ModifiedCommandLine = CommandLine;
ModifiedCommandLine->emplace_back(InputPath);
}
const std::vector<std::string> &FinalCommandLine =
ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;
return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer,
Controller, DC, FinalFS, /*ModuleName=*/std::nullopt);
}
bool DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, StringRef ModuleName) {
// Reset what might have been modified in the previous worker invocation.
BaseFS->setCurrentWorkingDirectory(WorkingDirectory);
// If we're scanning based on a module name alone, we don't expect the client
// to provide us with an input file. However, the driver really wants to have
// one. Let's just make it up to make the driver happy.
auto OverlayFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
SmallString<128> FakeInputPath;
// TODO: We should retry the creation if the path already exists.
llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath,
/*MakeAbsolute=*/false);
InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
OverlayFS->pushOverlay(InMemoryOverlay);
auto ModifiedCommandLine = CommandLine;
ModifiedCommandLine.emplace_back(FakeInputPath);
return scanDependencies(WorkingDirectory, ModifiedCommandLine, Consumer,
Controller, DC, OverlayFS, ModuleName);
}
DependencyActionController::~DependencyActionController() {}
Reference in New Issue View Git Blame Copy Permalink