Revert "[Inliner] Put inline history into IR as !inline_history metadata" (#190666)
Reverts llvm/llvm-project#190092 Crashes reported in https://github.com/llvm/llvm-project/pull/190092#issuecomment-4194546908
This commit is contained in:
Arthur Eubanks
GitHub
parent
40d3949162
commit
70d3dcaa64
@ -8852,25 +8852,6 @@ Example:
|
||||
!0 = !{ptr @a}
|
||||
!1 = !{ptr @b}
|
||||
|
||||
'``inline_history``' Metadata
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
The ``inline_history`` metadata may be attached to a call instruction. It
|
||||
indicates that the call instruction has been inlined from the referenced
|
||||
functions. The call itself should not be inlined if it is a call to any of the
|
||||
referenced functions since that could result in infinite inlining as we
|
||||
continually inline through mutually recursive functions.
|
||||
|
||||
This is intended to be added by and used by inliner passes.
|
||||
|
||||
The metadata operands must all be function pointers or ``null``. ``null`` can
|
||||
appear when the referenced function is erased from the module, e.g. an internal
|
||||
function that has had all calls to it inlined.
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
call void @foo(), !inline_history !0
|
||||
|
||||
!0 = !{ptr @bar, null, ptr @baz}
|
||||
|
||||
Module Flags Metadata
|
||||
=====================
|
||||
|
||||
@ -17,26 +17,26 @@ namespace llvm {
|
||||
class CallBase;
|
||||
template <typename Fn> class function_ref;
|
||||
|
||||
class InlineOrder {
|
||||
template <typename T> class InlineOrder {
|
||||
public:
|
||||
virtual ~InlineOrder() = default;
|
||||
|
||||
virtual size_t size() = 0;
|
||||
|
||||
virtual void push(CallBase *Elt) = 0;
|
||||
virtual void push(const T &Elt) = 0;
|
||||
|
||||
virtual CallBase *pop() = 0;
|
||||
virtual T pop() = 0;
|
||||
|
||||
virtual void erase_if(function_ref<bool(CallBase *)> Pred) = 0;
|
||||
virtual void erase_if(function_ref<bool(T)> Pred) = 0;
|
||||
|
||||
bool empty() { return !size(); }
|
||||
};
|
||||
|
||||
LLVM_ABI std::unique_ptr<InlineOrder>
|
||||
LLVM_ABI std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
|
||||
getDefaultInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
|
||||
ModuleAnalysisManager &MAM, Module &M);
|
||||
|
||||
LLVM_ABI std::unique_ptr<InlineOrder>
|
||||
LLVM_ABI std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
|
||||
getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
|
||||
ModuleAnalysisManager &MAM, Module &M);
|
||||
|
||||
@ -54,9 +54,10 @@ class PluginInlineOrderAnalysis
|
||||
public:
|
||||
LLVM_ABI static AnalysisKey Key;
|
||||
|
||||
typedef std::unique_ptr<InlineOrder> (*InlineOrderFactory)(
|
||||
FunctionAnalysisManager &FAM, const InlineParams &Params,
|
||||
ModuleAnalysisManager &MAM, Module &M);
|
||||
typedef std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>> (
|
||||
*InlineOrderFactory)(FunctionAnalysisManager &FAM,
|
||||
const InlineParams &Params,
|
||||
ModuleAnalysisManager &MAM, Module &M);
|
||||
|
||||
PluginInlineOrderAnalysis(InlineOrderFactory Factory) : Factory(Factory) {
|
||||
assert(Factory != nullptr &&
|
||||
|
||||
@ -60,4 +60,3 @@ LLVM_FIXED_MD_KIND(MD_alloc_token, "alloc_token", 45)
|
||||
LLVM_FIXED_MD_KIND(MD_implicit_ref, "implicit.ref", 46)
|
||||
LLVM_FIXED_MD_KIND(MD_nofpclass, "nofpclass", 47)
|
||||
LLVM_FIXED_MD_KIND(MD_call_target, "call_target", 48)
|
||||
LLVM_FIXED_MD_KIND(MD_inline_history, "inline_history", 49)
|
||||
|
||||
@ -18,7 +18,6 @@
|
||||
#define LLVM_TRANSFORMS_UTILS_CLONING_H
|
||||
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallPtrSet.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/ADT/Twine.h"
|
||||
#include "llvm/Analysis/AssumptionCache.h"
|
||||
@ -89,10 +88,6 @@ struct ClonedCodeInfo {
|
||||
/// check whether the main VMap mapping involves simplification or not.
|
||||
DenseMap<const Value *, const Value *> OrigVMap;
|
||||
|
||||
// Cloned calls that were originally an indirect call. They may be direct or
|
||||
// indirect after cloning.
|
||||
SmallPtrSet<const Value *, 4> OriginallyIndirectCalls;
|
||||
|
||||
ClonedCodeInfo() = default;
|
||||
|
||||
bool isSimplified(const Value *From, const Value *To) const {
|
||||
@ -230,12 +225,11 @@ LLVM_ABI void CloneFunctionBodyInto(
|
||||
ValueMaterializer *Materializer = nullptr,
|
||||
const MetadataPredicate *IdentityMD = nullptr);
|
||||
|
||||
LLVM_ABI void
|
||||
CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
|
||||
const Instruction *StartingInst,
|
||||
ValueToValueMapTy &VMap, bool ModuleLevelChanges,
|
||||
SmallVectorImpl<ReturnInst *> &Returns,
|
||||
const char *NameSuffix, ClonedCodeInfo &CodeInfo);
|
||||
LLVM_ABI void CloneAndPruneIntoFromInst(
|
||||
Function *NewFunc, const Function *OldFunc, const Instruction *StartingInst,
|
||||
ValueToValueMapTy &VMap, bool ModuleLevelChanges,
|
||||
SmallVectorImpl<ReturnInst *> &Returns, const char *NameSuffix = "",
|
||||
ClonedCodeInfo *CodeInfo = nullptr);
|
||||
|
||||
/// This works exactly like CloneFunctionInto,
|
||||
/// except that it does some simple constant prop and DCE on the fly. The
|
||||
@ -248,11 +242,10 @@ CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
|
||||
/// If ModuleLevelChanges is false, VMap contains no non-identity GlobalValue
|
||||
/// mappings.
|
||||
///
|
||||
LLVM_ABI void
|
||||
CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
|
||||
ValueToValueMapTy &VMap, bool ModuleLevelChanges,
|
||||
SmallVectorImpl<ReturnInst *> &Returns,
|
||||
const char *NameSuffix, ClonedCodeInfo &CodeInfo);
|
||||
LLVM_ABI void CloneAndPruneFunctionInto(
|
||||
Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap,
|
||||
bool ModuleLevelChanges, SmallVectorImpl<ReturnInst *> &Returns,
|
||||
const char *NameSuffix = "", ClonedCodeInfo *CodeInfo = nullptr);
|
||||
|
||||
/// This class captures the data input to the InlineFunction call, and records
|
||||
/// the auxiliary results produced by it.
|
||||
@ -318,7 +311,6 @@ LLVM_ABI void InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
bool MergeAttributes = false,
|
||||
AAResults *CalleeAAR = nullptr,
|
||||
bool InsertLifetime = true,
|
||||
bool TrackInlineHistory = false,
|
||||
Function *ForwardVarArgsTo = nullptr,
|
||||
OptimizationRemarkEmitter *ORE = nullptr);
|
||||
|
||||
@ -348,7 +340,6 @@ LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
bool MergeAttributes = false,
|
||||
AAResults *CalleeAAR = nullptr,
|
||||
bool InsertLifetime = true,
|
||||
bool TrackInlineHistory = false,
|
||||
Function *ForwardVarArgsTo = nullptr,
|
||||
OptimizationRemarkEmitter *ORE = nullptr);
|
||||
|
||||
@ -361,7 +352,6 @@ LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
bool MergeAttributes = false,
|
||||
AAResults *CalleeAAR = nullptr,
|
||||
bool InsertLifetime = true,
|
||||
bool TrackInlineHistory = false,
|
||||
Function *ForwardVarArgsTo = nullptr,
|
||||
OptimizationRemarkEmitter *ORE = nullptr);
|
||||
|
||||
@ -443,6 +433,14 @@ LLVM_ABI void cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
|
||||
LLVM_ABI void cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,
|
||||
Instruction *IStart, Instruction *IEnd,
|
||||
LLVMContext &Context, StringRef Ext);
|
||||
/// Check if Function F appears in the inline history chain.
|
||||
/// InlineHistory is a vector of (Function, ParentHistoryID) pairs.
|
||||
/// Returns true if F was already inlined in the chain leading to
|
||||
/// InlineHistoryID.
|
||||
LLVM_ABI bool
|
||||
inlineHistoryIncludes(Function *F, int InlineHistoryID,
|
||||
ArrayRef<std::pair<Function *, int>> InlineHistory);
|
||||
|
||||
} // end namespace llvm
|
||||
|
||||
#endif // LLVM_TRANSFORMS_UTILS_CLONING_H
|
||||
|
||||
@ -199,7 +199,10 @@ private:
|
||||
int Cost = INT_MAX;
|
||||
};
|
||||
|
||||
template <typename PriorityT> class PriorityInlineOrder : public InlineOrder {
|
||||
template <typename PriorityT>
|
||||
class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
|
||||
using T = std::pair<CallBase *, int>;
|
||||
|
||||
bool hasLowerPriority(const CallBase *L, const CallBase *R) const {
|
||||
const auto I1 = Priorities.find(L);
|
||||
const auto I2 = Priorities.find(R);
|
||||
@ -240,21 +243,31 @@ public:
|
||||
|
||||
size_t size() override { return Heap.size(); }
|
||||
|
||||
void push(CallBase *CB) override {
|
||||
void push(const T &Elt) override {
|
||||
CallBase *CB = Elt.first;
|
||||
const int InlineHistoryID = Elt.second;
|
||||
|
||||
Heap.push_back(CB);
|
||||
Priorities[CB] = PriorityT(CB, FAM, Params);
|
||||
std::push_heap(Heap.begin(), Heap.end(), isLess);
|
||||
InlineHistoryMap[CB] = InlineHistoryID;
|
||||
}
|
||||
|
||||
CallBase *pop() override {
|
||||
T pop() override {
|
||||
assert(size() > 0);
|
||||
pop_heap_adjust();
|
||||
|
||||
return Heap.pop_back_val();
|
||||
CallBase *CB = Heap.pop_back_val();
|
||||
T Result = std::make_pair(CB, InlineHistoryMap[CB]);
|
||||
InlineHistoryMap.erase(CB);
|
||||
return Result;
|
||||
}
|
||||
|
||||
void erase_if(function_ref<bool(CallBase *)> Pred) override {
|
||||
llvm::erase_if(Heap, Pred);
|
||||
void erase_if(function_ref<bool(T)> Pred) override {
|
||||
auto PredWrapper = [=](CallBase *CB) -> bool {
|
||||
return Pred(std::make_pair(CB, InlineHistoryMap[CB]));
|
||||
};
|
||||
llvm::erase_if(Heap, PredWrapper);
|
||||
std::make_heap(Heap.begin(), Heap.end(), isLess);
|
||||
}
|
||||
|
||||
@ -271,7 +284,7 @@ private:
|
||||
|
||||
AnalysisKey llvm::PluginInlineOrderAnalysis::Key;
|
||||
|
||||
std::unique_ptr<InlineOrder>
|
||||
std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
|
||||
llvm::getDefaultInlineOrder(FunctionAnalysisManager &FAM,
|
||||
const InlineParams &Params,
|
||||
ModuleAnalysisManager &MAM, Module &M) {
|
||||
@ -296,10 +309,9 @@ llvm::getDefaultInlineOrder(FunctionAnalysisManager &FAM,
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::unique_ptr<InlineOrder> llvm::getInlineOrder(FunctionAnalysisManager &FAM,
|
||||
const InlineParams &Params,
|
||||
ModuleAnalysisManager &MAM,
|
||||
Module &M) {
|
||||
std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
|
||||
llvm::getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
|
||||
ModuleAnalysisManager &MAM, Module &M) {
|
||||
if (MAM.isPassRegistered<PluginInlineOrderAnalysis>()) {
|
||||
LLVM_DEBUG(dbgs() << " Current used priority: plugin ---- \n");
|
||||
return MAM.getResult<PluginInlineOrderAnalysis>(M).Factory(FAM, Params, MAM,
|
||||
|
||||
@ -549,7 +549,6 @@ private:
|
||||
void visitAccessGroupMetadata(const MDNode *MD);
|
||||
void visitCapturesMetadata(Instruction &I, const MDNode *Captures);
|
||||
void visitAllocTokenMetadata(Instruction &I, MDNode *MD);
|
||||
void visitInlineHistoryMetadata(Instruction &I, MDNode *MD);
|
||||
|
||||
template <class Ty> bool isValidMetadataArray(const MDTuple &N);
|
||||
#define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N);
|
||||
@ -5620,18 +5619,6 @@ void Verifier::visitAllocTokenMetadata(Instruction &I, MDNode *MD) {
|
||||
"expected integer constant", MD);
|
||||
}
|
||||
|
||||
void Verifier::visitInlineHistoryMetadata(Instruction &I, MDNode *MD) {
|
||||
Check(isa<CallBase>(I), "!inline_history should only exist on calls", &I);
|
||||
for (Metadata *Op : MD->operands()) {
|
||||
// Can be null when a function is erased.
|
||||
if (!Op)
|
||||
continue;
|
||||
Check(isa<ValueAsMetadata>(Op) &&
|
||||
isa<Function>(cast<ValueAsMetadata>(Op)->getValue()),
|
||||
"!inline_history operands must be functions or null", MD);
|
||||
}
|
||||
}
|
||||
|
||||
/// verifyInstruction - Verify that an instruction is well formed.
|
||||
///
|
||||
void Verifier::visitInstruction(Instruction &I) {
|
||||
@ -5864,9 +5851,6 @@ void Verifier::visitInstruction(Instruction &I) {
|
||||
if (MDNode *MD = I.getMetadata(LLVMContext::MD_alloc_token))
|
||||
visitAllocTokenMetadata(I, MD);
|
||||
|
||||
if (MDNode *MD = I.getMetadata(LLVMContext::MD_inline_history))
|
||||
visitInlineHistoryMetadata(I, MD);
|
||||
|
||||
if (MDNode *N = I.getDebugLoc().getAsMDNode()) {
|
||||
CheckDI(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N);
|
||||
visitMDNode(*N, AreDebugLocsAllowed::Yes);
|
||||
|
||||
@ -53,9 +53,8 @@ bool AlwaysInlineImpl(
|
||||
BasicBlock *Block = CB.getParent();
|
||||
|
||||
InlineFunctionInfo IFI(GetAssumptionCache, &PSI);
|
||||
InlineResult Res = InlineFunction(
|
||||
CB, IFI, /*MergeAttributes=*/true, &GetAAR(Callee), InsertLifetime,
|
||||
/*TrackInlineHistory=*/NewCallSites != nullptr);
|
||||
InlineResult Res = InlineFunction(CB, IFI, /*MergeAttributes=*/true,
|
||||
&GetAAR(Callee), InsertLifetime);
|
||||
if (!Res.isSuccess()) {
|
||||
ORE.emit([&]() {
|
||||
return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
|
||||
@ -141,7 +140,8 @@ bool AlwaysInlineImpl(
|
||||
continue;
|
||||
|
||||
// Detect recursion.
|
||||
if (Callee == F) {
|
||||
if (Callee == F ||
|
||||
inlineHistoryIncludes(Callee, InlineHistoryID, InlineHistory)) {
|
||||
ORE.emit([&]() {
|
||||
return OptimizationRemarkMissed("inline", "NotInlined",
|
||||
CB->getDebugLoc(), CB->getParent())
|
||||
|
||||
@ -231,7 +231,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
// this model, but it is uniformly spread across all the functions in the SCC
|
||||
// and eventually they all become too large to inline, rather than
|
||||
// incrementally maknig a single function grow in a super linear fashion.
|
||||
SmallVector<CallBase *, 16> Calls;
|
||||
SmallVector<std::pair<CallBase *, int>, 16> Calls;
|
||||
|
||||
// Populate the initial list of calls in this SCC.
|
||||
for (auto &N : InitialC) {
|
||||
@ -246,7 +246,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
if (auto *CB = dyn_cast<CallBase>(&I))
|
||||
if (Function *Callee = CB->getCalledFunction()) {
|
||||
if (!Callee->isDeclaration())
|
||||
Calls.push_back(CB);
|
||||
Calls.push_back({CB, -1});
|
||||
else if (!isa<IntrinsicInst>(I)) {
|
||||
using namespace ore;
|
||||
setInlineRemark(*CB, "unavailable definition");
|
||||
@ -269,6 +269,12 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
if (Calls.empty())
|
||||
return PreservedAnalyses::all();
|
||||
|
||||
// When inlining a callee produces new call sites, we want to keep track of
|
||||
// the fact that they were inlined from the callee. This allows us to avoid
|
||||
// infinite inlining in some obscure cases. To represent this, we use an
|
||||
// index into the InlineHistory vector.
|
||||
SmallVector<std::pair<Function *, int>, 16> InlineHistory;
|
||||
|
||||
// Track a set vector of inlined callees so that we can augment the caller
|
||||
// with all of their edges in the call graph before pruning out the ones that
|
||||
// got simplified away.
|
||||
@ -289,7 +295,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
// have the same caller, so we first set up some shared infrastructure for
|
||||
// this caller. We also do any pruning we can at this layer on the caller
|
||||
// alone.
|
||||
Function &F = *Calls[I]->getCaller();
|
||||
Function &F = *Calls[I].first->getCaller();
|
||||
LazyCallGraph::Node &N = *CG.lookup(F);
|
||||
if (CG.lookupSCC(N) != C)
|
||||
continue;
|
||||
@ -306,17 +312,29 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
// We bail out as soon as the caller has to change so we can update the
|
||||
// call graph and prepare the context of that new caller.
|
||||
bool DidInline = false;
|
||||
for (; I < (int)Calls.size() && Calls[I]->getCaller() == &F; ++I) {
|
||||
CallBase *CB = Calls[I];
|
||||
for (; I < (int)Calls.size() && Calls[I].first->getCaller() == &F; ++I) {
|
||||
auto &P = Calls[I];
|
||||
CallBase *CB = P.first;
|
||||
const int InlineHistoryID = P.second;
|
||||
Function &Callee = *CB->getCalledFunction();
|
||||
|
||||
if (InlineHistoryID != -1 &&
|
||||
inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
|
||||
LLVM_DEBUG(dbgs() << "Skipping inlining due to history: " << F.getName()
|
||||
<< " -> " << Callee.getName() << "\n");
|
||||
setInlineRemark(*CB, "recursive");
|
||||
// Set noinline so that we don't forget this decision across CGSCC
|
||||
// iterations.
|
||||
CB->setIsNoInline();
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this inlining may repeat breaking an SCC apart that has
|
||||
// already been split once before. In that case, inlining here may
|
||||
// trigger infinite inlining, much like is prevented within the inliner
|
||||
// itself by the InlineHistory above, but spread across CGSCC iterations
|
||||
// and thus hidden from the full inline history.
|
||||
LazyCallGraph::Node &CalleeN = *CG.lookup(Callee);
|
||||
LazyCallGraph::SCC *CalleeSCC = CG.lookupSCC(CalleeN);
|
||||
LazyCallGraph::SCC *CalleeSCC = CG.lookupSCC(*CG.lookup(Callee));
|
||||
if (CalleeSCC == C && UR.InlinedInternalEdges.count({&N, C})) {
|
||||
LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node "
|
||||
"previously split out of this SCC by inlining: "
|
||||
@ -349,20 +367,9 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
&FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
|
||||
&FAM.getResult<BlockFrequencyAnalysis>(Callee));
|
||||
|
||||
// For compile time reasons we try to only track inline history for the
|
||||
// calls where it may actually prevent inlining, which is inlining through
|
||||
// an SCC. This can happen if the callee is in a non-trivial SCC/RefSCC,
|
||||
// or if an inlined call site was an indirect call, which can be
|
||||
// devirtualized to call any target by replacing the indirectly called
|
||||
// function with a function pointer referenced by the caller. The indirect
|
||||
// call case is handled within InlineFunction.
|
||||
bool TrackInlineHistory =
|
||||
CalleeSCC->size() != 1 || CalleeSCC->getOuterRefSCC().size() != 1;
|
||||
|
||||
InlineResult IR = InlineFunction(
|
||||
*CB, IFI, /*MergeAttributes=*/true,
|
||||
&FAM.getResult<AAManager>(*CB->getCaller()), /*InsertLifetime=*/true,
|
||||
TrackInlineHistory, nullptr,
|
||||
&FAM.getResult<AAManager>(*CB->getCaller()), true, nullptr,
|
||||
&FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB->getCaller()));
|
||||
if (!IR.isSuccess()) {
|
||||
Advice->recordUnsuccessfulInlining(IR);
|
||||
@ -381,6 +388,9 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
|
||||
// Add any new callsites to defined functions to the worklist.
|
||||
if (!IFI.InlinedCallSites.empty()) {
|
||||
int NewHistoryID = InlineHistory.size();
|
||||
InlineHistory.push_back({&Callee, InlineHistoryID});
|
||||
|
||||
for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
|
||||
Function *NewCallee = ICB->getCalledFunction();
|
||||
assert(!(NewCallee && NewCallee->isIntrinsic()) &&
|
||||
@ -395,7 +405,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
}
|
||||
if (NewCallee) {
|
||||
if (!NewCallee->isDeclaration()) {
|
||||
Calls.push_back(ICB);
|
||||
Calls.push_back({ICB, NewHistoryID});
|
||||
// Continually inlining through an SCC can result in huge compile
|
||||
// times and bloated code since we arbitrarily stop at some point
|
||||
// when the inliner decides it's not profitable to inline anymore.
|
||||
@ -427,11 +437,12 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
|
||||
if (Callee.isDiscardableIfUnused() && Callee.hasZeroLiveUses() &&
|
||||
!CG.isLibFunction(Callee)) {
|
||||
if (Callee.hasLocalLinkage() || !Callee.hasComdat()) {
|
||||
Calls.erase(std::remove_if(Calls.begin() + I + 1, Calls.end(),
|
||||
[&](const CallBase *CB) {
|
||||
return CB->getCaller() == &Callee;
|
||||
}),
|
||||
Calls.end());
|
||||
Calls.erase(
|
||||
std::remove_if(Calls.begin() + I + 1, Calls.end(),
|
||||
[&](const std::pair<CallBase *, int> &Call) {
|
||||
return Call.first->getCaller() == &Callee;
|
||||
}),
|
||||
Calls.end());
|
||||
|
||||
// Report inlining decision BEFORE deleting function contents, so we
|
||||
// can still access e.g. the DebugLoc
|
||||
|
||||
@ -144,7 +144,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
|
||||
if (auto *CB = dyn_cast<CallBase>(&I)) {
|
||||
if (Function *Callee = CB->getCalledFunction()) {
|
||||
if (!Callee->isDeclaration())
|
||||
Calls->push(CB);
|
||||
Calls->push({CB, -1});
|
||||
else if (!isa<IntrinsicInst>(I)) {
|
||||
using namespace ore;
|
||||
setInlineRemark(*CB, "unavailable definition");
|
||||
@ -166,18 +166,26 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
|
||||
}
|
||||
for (auto &[CB, Target] : ICPCandidates) {
|
||||
if (auto *DirectCB = promoteCallWithIfThenElse(*CB, *Target, CtxProf))
|
||||
Calls->push(DirectCB);
|
||||
Calls->push({DirectCB, -1});
|
||||
}
|
||||
if (Calls->empty())
|
||||
return PreservedAnalyses::all();
|
||||
|
||||
// When inlining a callee produces new call sites, we want to keep track of
|
||||
// the fact that they were inlined from the callee. This allows us to avoid
|
||||
// infinite inlining in some obscure cases. To represent this, we use an
|
||||
// index into the InlineHistory vector.
|
||||
SmallVector<std::pair<Function *, int>, 16> InlineHistory;
|
||||
|
||||
// Track the dead functions to delete once finished with inlining calls. We
|
||||
// defer deleting these to make it easier to handle the call graph updates.
|
||||
SmallVector<Function *, 4> DeadFunctions;
|
||||
|
||||
// Loop forward over all of the calls.
|
||||
while (!Calls->empty()) {
|
||||
CallBase *CB = Calls->pop();
|
||||
auto P = Calls->pop();
|
||||
CallBase *CB = P.first;
|
||||
const int InlineHistoryID = P.second;
|
||||
Function &F = *CB->getCaller();
|
||||
Function &Callee = *CB->getCalledFunction();
|
||||
|
||||
@ -190,6 +198,12 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
|
||||
return FAM.getResult<AssumptionAnalysis>(F);
|
||||
};
|
||||
|
||||
if (InlineHistoryID != -1 &&
|
||||
inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
|
||||
setInlineRemark(*CB, "recursive");
|
||||
continue;
|
||||
}
|
||||
|
||||
auto Advice = Advisor.getAdvice(*CB, /*OnlyMandatory*/ false);
|
||||
// Check whether we want to inline this callsite.
|
||||
if (!Advice->isInliningRecommended()) {
|
||||
@ -220,6 +234,9 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
|
||||
|
||||
// Add any new callsites to defined functions to the worklist.
|
||||
if (!IFI.InlinedCallSites.empty()) {
|
||||
int NewHistoryID = InlineHistory.size();
|
||||
InlineHistory.push_back({&Callee, InlineHistoryID});
|
||||
|
||||
for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
|
||||
Function *NewCallee = ICB->getCalledFunction();
|
||||
if (!NewCallee) {
|
||||
@ -234,7 +251,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
|
||||
}
|
||||
if (NewCallee)
|
||||
if (!NewCallee->isDeclaration())
|
||||
Calls->push(ICB);
|
||||
Calls->push({ICB, NewHistoryID});
|
||||
}
|
||||
}
|
||||
|
||||
@ -249,8 +266,9 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
|
||||
Callee.removeDeadConstantUsers();
|
||||
// if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
|
||||
if (Callee.use_empty() && !isKnownLibFunction(Callee, GetTLI(Callee))) {
|
||||
Calls->erase_if(
|
||||
[&](const CallBase *CB) { return CB->getCaller() == &Callee; });
|
||||
Calls->erase_if([&](const std::pair<CallBase *, int> &Call) {
|
||||
return Call.first->getCaller() == &Callee;
|
||||
});
|
||||
|
||||
// Report inlining decision BEFORE deleting function contents, so we
|
||||
// can still access e.g. the DebugLoc
|
||||
|
||||
@ -1372,8 +1372,7 @@ bool PartialInlinerImpl::tryPartialInline(FunctionCloner &Cloner) {
|
||||
InlineFunctionInfo IFI(GetAssumptionCache, &PSI);
|
||||
// We can only forward varargs when we outlined a single region, else we
|
||||
// bail on vararg functions.
|
||||
if (!InlineFunction(*CB, IFI, /*MergeAttributes=*/false, nullptr,
|
||||
/*InsertLifetime=*/true, /*TrackInlineHistory=*/true,
|
||||
if (!InlineFunction(*CB, IFI, /*MergeAttributes=*/false, nullptr, true,
|
||||
(Cloner.ClonedOI ? Cloner.OutlinedFunctions.back().first
|
||||
: nullptr))
|
||||
.isSuccess())
|
||||
|
||||
@ -433,7 +433,7 @@ struct PruningFunctionCloner {
|
||||
ValueToValueMapTy &VMap;
|
||||
bool ModuleLevelChanges;
|
||||
const char *NameSuffix;
|
||||
ClonedCodeInfo &CodeInfo;
|
||||
ClonedCodeInfo *CodeInfo;
|
||||
bool HostFuncIsStrictFP;
|
||||
|
||||
Instruction *cloneInstruction(BasicBlock::const_iterator II);
|
||||
@ -441,7 +441,7 @@ struct PruningFunctionCloner {
|
||||
public:
|
||||
PruningFunctionCloner(Function *newFunc, const Function *oldFunc,
|
||||
ValueToValueMapTy &valueMap, bool moduleLevelChanges,
|
||||
const char *nameSuffix, ClonedCodeInfo &codeInfo)
|
||||
const char *nameSuffix, ClonedCodeInfo *codeInfo)
|
||||
: NewFunc(newFunc), OldFunc(oldFunc), VMap(valueMap),
|
||||
ModuleLevelChanges(moduleLevelChanges), NameSuffix(nameSuffix),
|
||||
CodeInfo(codeInfo) {
|
||||
@ -615,9 +615,6 @@ void PruningFunctionCloner::CloneBlock(
|
||||
}
|
||||
}
|
||||
|
||||
if (auto *CB = dyn_cast<CallBase>(II); CB && CB->isIndirectCall())
|
||||
CodeInfo.OriginallyIndirectCalls.insert(NewInst);
|
||||
|
||||
if (II->hasName())
|
||||
NewInst->setName(II->getName() + NameSuffix);
|
||||
VMap[&*II] = NewInst; // Add instruction map to value.
|
||||
@ -629,10 +626,12 @@ void PruningFunctionCloner::CloneBlock(
|
||||
|
||||
CloneDbgRecordsToHere(NewInst, II);
|
||||
|
||||
CodeInfo.OrigVMap[&*II] = NewInst;
|
||||
if (auto *CB = dyn_cast<CallBase>(&*II))
|
||||
if (CB->hasOperandBundles())
|
||||
CodeInfo.OperandBundleCallSites.push_back(NewInst);
|
||||
if (CodeInfo) {
|
||||
CodeInfo->OrigVMap[&*II] = NewInst;
|
||||
if (auto *CB = dyn_cast<CallBase>(&*II))
|
||||
if (CB->hasOperandBundles())
|
||||
CodeInfo->OperandBundleCallSites.push_back(NewInst);
|
||||
}
|
||||
|
||||
if (const AllocaInst *AI = dyn_cast<AllocaInst>(II)) {
|
||||
if (isa<ConstantInt>(AI->getArraySize()))
|
||||
@ -691,10 +690,12 @@ void PruningFunctionCloner::CloneBlock(
|
||||
|
||||
VMap[OldTI] = NewInst; // Add instruction map to value.
|
||||
|
||||
CodeInfo.OrigVMap[OldTI] = NewInst;
|
||||
if (auto *CB = dyn_cast<CallBase>(OldTI))
|
||||
if (CB->hasOperandBundles())
|
||||
CodeInfo.OperandBundleCallSites.push_back(NewInst);
|
||||
if (CodeInfo) {
|
||||
CodeInfo->OrigVMap[OldTI] = NewInst;
|
||||
if (auto *CB = dyn_cast<CallBase>(OldTI))
|
||||
if (CB->hasOperandBundles())
|
||||
CodeInfo->OperandBundleCallSites.push_back(NewInst);
|
||||
}
|
||||
|
||||
// Recursively clone any reachable successor blocks.
|
||||
append_range(ToClone, successors(BB->getTerminator()));
|
||||
@ -707,11 +708,13 @@ void PruningFunctionCloner::CloneBlock(
|
||||
CloneDbgRecordsToHere(NewInst, OldTI->getIterator());
|
||||
}
|
||||
|
||||
CodeInfo.ContainsCalls |= hasCalls;
|
||||
CodeInfo.ContainsMemProfMetadata |= hasMemProfMetadata;
|
||||
CodeInfo.ContainsDynamicAllocas |= hasDynamicAllocas;
|
||||
CodeInfo.ContainsDynamicAllocas |=
|
||||
hasStaticAllocas && BB != &BB->getParent()->front();
|
||||
if (CodeInfo) {
|
||||
CodeInfo->ContainsCalls |= hasCalls;
|
||||
CodeInfo->ContainsMemProfMetadata |= hasMemProfMetadata;
|
||||
CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;
|
||||
CodeInfo->ContainsDynamicAllocas |=
|
||||
hasStaticAllocas && BB != &BB->getParent()->front();
|
||||
}
|
||||
}
|
||||
|
||||
/// This works like CloneAndPruneFunctionInto, except that it does not clone the
|
||||
@ -723,7 +726,7 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
|
||||
bool ModuleLevelChanges,
|
||||
SmallVectorImpl<ReturnInst *> &Returns,
|
||||
const char *NameSuffix,
|
||||
ClonedCodeInfo &CodeInfo) {
|
||||
ClonedCodeInfo *CodeInfo) {
|
||||
assert(NameSuffix && "NameSuffix cannot be null!");
|
||||
|
||||
ValueMapTypeRemapper *TypeMapper = nullptr;
|
||||
@ -1003,12 +1006,10 @@ void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
|
||||
/// constant arguments cause a significant amount of code in the callee to be
|
||||
/// dead. Since this doesn't produce an exact copy of the input, it can't be
|
||||
/// used for things like CloneFunction or CloneModule.
|
||||
void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
|
||||
ValueToValueMapTy &VMap,
|
||||
bool ModuleLevelChanges,
|
||||
SmallVectorImpl<ReturnInst *> &Returns,
|
||||
const char *NameSuffix,
|
||||
ClonedCodeInfo &CodeInfo) {
|
||||
void llvm::CloneAndPruneFunctionInto(
|
||||
Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap,
|
||||
bool ModuleLevelChanges, SmallVectorImpl<ReturnInst *> &Returns,
|
||||
const char *NameSuffix, ClonedCodeInfo *CodeInfo) {
|
||||
CloneAndPruneIntoFromInst(NewFunc, OldFunc, &OldFunc->front().front(), VMap,
|
||||
ModuleLevelChanges, Returns, NameSuffix, CodeInfo);
|
||||
}
|
||||
|
||||
@ -2412,15 +2412,13 @@ remapIndices(Function &Caller, BasicBlock *StartBB,
|
||||
// Updating the contextual profile after an inlining means, at a high level,
|
||||
// copying over the data of the callee, **intentionally without any value
|
||||
// scaling**, and copying over the callees of the inlined callee.
|
||||
llvm::InlineResult
|
||||
llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
PGOContextualProfile &CtxProf, bool MergeAttributes,
|
||||
AAResults *CalleeAAR, bool InsertLifetime,
|
||||
bool TrackInlineHistory, Function *ForwardVarArgsTo,
|
||||
OptimizationRemarkEmitter *ORE) {
|
||||
llvm::InlineResult llvm::InlineFunction(
|
||||
CallBase &CB, InlineFunctionInfo &IFI, PGOContextualProfile &CtxProf,
|
||||
bool MergeAttributes, AAResults *CalleeAAR, bool InsertLifetime,
|
||||
Function *ForwardVarArgsTo, OptimizationRemarkEmitter *ORE) {
|
||||
if (!CtxProf.isInSpecializedModule())
|
||||
return InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
|
||||
TrackInlineHistory, ForwardVarArgsTo, ORE);
|
||||
ForwardVarArgsTo, ORE);
|
||||
|
||||
auto &Caller = *CB.getCaller();
|
||||
auto &Callee = *CB.getCalledFunction();
|
||||
@ -2438,7 +2436,7 @@ llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
const auto NumCalleeCallsites = CtxProf.getNumCallsites(Callee);
|
||||
|
||||
auto Ret = InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
|
||||
TrackInlineHistory, ForwardVarArgsTo, ORE);
|
||||
ForwardVarArgsTo, ORE);
|
||||
if (!Ret.isSuccess())
|
||||
return Ret;
|
||||
|
||||
@ -2524,18 +2522,6 @@ llvm::InlineResult llvm::CanInlineCallSite(const CallBase &CB,
|
||||
CalledFunc->isDeclaration()) // call!
|
||||
return InlineResult::failure("external or indirect");
|
||||
|
||||
// Don't inline if we've already inlined this callee through this call site
|
||||
// before to prevent infinite inlining through mutually recursive functions.
|
||||
if (MDNode *InlineHistory = CB.getMetadata(LLVMContext::MD_inline_history)) {
|
||||
for (const auto &Op : InlineHistory->operands()) {
|
||||
if (auto *MD = dyn_cast_or_null<ValueAsMetadata>(Op)) {
|
||||
if (MD->getValue() == CalledFunc) {
|
||||
return InlineResult::failure("inline history");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The inliner does not know how to inline through calls with operand bundles
|
||||
// in general ...
|
||||
if (CB.hasOperandBundles()) {
|
||||
@ -2661,8 +2647,7 @@ llvm::InlineResult llvm::CanInlineCallSite(const CallBase &CB,
|
||||
/// function by one level.
|
||||
void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
bool MergeAttributes, AAResults *CalleeAAR,
|
||||
bool InsertLifetime, bool TrackInlineHistory,
|
||||
Function *ForwardVarArgsTo,
|
||||
bool InsertLifetime, Function *ForwardVarArgsTo,
|
||||
OptimizationRemarkEmitter *ORE) {
|
||||
BasicBlock *OrigBB = CB.getParent();
|
||||
Function *Caller = OrigBB->getParent();
|
||||
@ -2783,7 +2768,7 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
// happy with whatever the cloner can do.
|
||||
CloneAndPruneFunctionInto(Caller, CalledFunc, VMap,
|
||||
/*ModuleLevelChanges=*/false, Returns, ".i",
|
||||
InlinedFunctionInfo);
|
||||
&InlinedFunctionInfo);
|
||||
// Remember the first block that is newly cloned over.
|
||||
FirstNewBlock = LastBlock; ++FirstNewBlock;
|
||||
|
||||
@ -3284,35 +3269,6 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
IFI.InlinedCallSites.push_back(CB);
|
||||
}
|
||||
|
||||
for (CallBase *ICB : IFI.InlinedCallSites) {
|
||||
// We only track inline history if requested, or if the inlined call site
|
||||
// was originally an indirect call (it may have become a direct call
|
||||
// during inlining).
|
||||
if (TrackInlineHistory ||
|
||||
InlinedFunctionInfo.OriginallyIndirectCalls.contains(ICB)) {
|
||||
// !inline_history is {Callee, CB.inline_history, ICB.inline_history}.
|
||||
// Metadata nodes may be null if the referenced function was erased from
|
||||
// the module.
|
||||
SmallVector<Metadata *, 4> History;
|
||||
History.push_back(ValueAsMetadata::get(CalledFunc));
|
||||
if (MDNode *CBHistory = CB.getMetadata(LLVMContext::MD_inline_history)) {
|
||||
for (const auto &Op : CBHistory->operands()) {
|
||||
if (Op)
|
||||
History.push_back(Op.get());
|
||||
}
|
||||
}
|
||||
if (MDNode *CBHistory =
|
||||
ICB->getMetadata(LLVMContext::MD_inline_history)) {
|
||||
for (const auto &Op : CBHistory->operands()) {
|
||||
if (Op)
|
||||
History.push_back(Op.get());
|
||||
}
|
||||
}
|
||||
MDNode *NewHistory = MDNode::get(Caller->getContext(), History);
|
||||
ICB->setMetadata(LLVMContext::MD_inline_history, NewHistory);
|
||||
}
|
||||
}
|
||||
|
||||
// If we cloned in _exactly one_ basic block, and if that block ends in a
|
||||
// return instruction, we splice the body of the inlined callee directly into
|
||||
// the calling basic block.
|
||||
@ -3518,15 +3474,30 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc);
|
||||
}
|
||||
|
||||
llvm::InlineResult llvm::InlineFunction(
|
||||
CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes,
|
||||
AAResults *CalleeAAR, bool InsertLifetime, bool TrackInlineHistory,
|
||||
Function *ForwardVarArgsTo, OptimizationRemarkEmitter *ORE) {
|
||||
llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
|
||||
bool MergeAttributes,
|
||||
AAResults *CalleeAAR,
|
||||
bool InsertLifetime,
|
||||
Function *ForwardVarArgsTo,
|
||||
OptimizationRemarkEmitter *ORE) {
|
||||
llvm::InlineResult Result = CanInlineCallSite(CB, IFI);
|
||||
if (Result.isSuccess()) {
|
||||
InlineFunctionImpl(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
|
||||
TrackInlineHistory, ForwardVarArgsTo, ORE);
|
||||
ForwardVarArgsTo, ORE);
|
||||
}
|
||||
|
||||
return Result;
|
||||
}
|
||||
|
||||
bool llvm::inlineHistoryIncludes(
|
||||
Function *F, int InlineHistoryID,
|
||||
ArrayRef<std::pair<Function *, int>> InlineHistory) {
|
||||
while (InlineHistoryID != -1) {
|
||||
assert(unsigned(InlineHistoryID) < InlineHistory.size() &&
|
||||
"Invalid inline history ID");
|
||||
if (InlineHistory[InlineHistoryID].first == F)
|
||||
return true;
|
||||
InlineHistoryID = InlineHistory[InlineHistoryID].second;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
; Test that the debug location is preserved when rewriting an inlined call as an invoke
|
||||
|
||||
; CHECK: invoke void @test()
|
||||
; CHECK-NEXT: to label {{.*}} unwind label {{.*}}, !dbg [[INL_LOC:![0-9]+]]
|
||||
; CHECK-NEXT: to label {{.*}} unwind label {{.*}}, !dbg [[INL_LOC:!.*]]
|
||||
; CHECK: [[SP:.*]] = distinct !DISubprogram(
|
||||
; CHECK: [[INL_LOC]] = !DILocation(line: 1, scope: [[SP]], inlinedAt: [[INL_AT:.*]])
|
||||
; CHECK: [[INL_AT]] = distinct !DILocation(line: 2, scope: [[SP]])
|
||||
|
||||
@ -1,25 +0,0 @@
|
||||
; RUN: opt -passes='cgscc(inline,function(instcombine))' -S < %s | FileCheck %s
|
||||
|
||||
; Check that devirtualization of an indirect call to itself doesn't cause
|
||||
; infinite inlining. Note that we need the devirtualization to happen in
|
||||
; simplification between inlining runs or else InlineCost will substitute %p
|
||||
; with @p and see that the call is a recursive call and bail.
|
||||
|
||||
define void @p(ptr %p, i64 %x) {
|
||||
%a = alloca ptr
|
||||
store ptr %p, ptr %a
|
||||
%g = getelementptr i8, ptr %a, i64 %x
|
||||
%b = load ptr, ptr %g
|
||||
call void %b(ptr %p, i64 %x)
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @q() {
|
||||
; CHECK-LABEL: define void @q() {
|
||||
; CHECK-NEXT: call void @p({{.*}}), !inline_history [[HISTORY:![0-9]+]]
|
||||
; CHECK-NEXT: ret void
|
||||
call void @p(ptr @p, i64 0)
|
||||
ret void
|
||||
}
|
||||
|
||||
; CHECK: [[HISTORY]] = !{ptr @p}
|
||||
32
llvm/test/Transforms/Inline/inline-history-noinline.ll
Normal file
32
llvm/test/Transforms/Inline/inline-history-noinline.ll
Normal file
@ -0,0 +1,32 @@
|
||||
; RUN: opt -passes=inline -S < %s | FileCheck %s
|
||||
|
||||
; This will inline @f1 into @a, causing two new calls to @f2, which will get inlined for two calls to @f1.
|
||||
; The inline history should stop recursive inlining here, and make sure to mark the inlined calls as noinline so we don't repeat the inlining later on when @a gets inlined into @b.
|
||||
|
||||
define internal void @f1(ptr %p) {
|
||||
call void %p(ptr @f1)
|
||||
ret void
|
||||
}
|
||||
|
||||
define internal void @f2(ptr %p) {
|
||||
call void %p(ptr @f2)
|
||||
call void %p(ptr @f2)
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @b() {
|
||||
; CHECK-LABEL: define void @b() {
|
||||
; CHECK-NEXT: call void @f1(ptr @f2) #[[NOINLINE:[0-9]+]]
|
||||
; CHECK-NEXT: call void @f1(ptr @f2) #[[NOINLINE]]
|
||||
; CHECK-NEXT: ret void
|
||||
;
|
||||
call void @a()
|
||||
ret void
|
||||
}
|
||||
|
||||
define internal void @a() {
|
||||
call void @f1(ptr @f2)
|
||||
ret void
|
||||
}
|
||||
|
||||
; CHECK: [[NOINLINE]] = { noinline }
|
||||
@ -1,102 +0,0 @@
|
||||
; RUN: opt -passes=inline -S < %s | FileCheck %s
|
||||
|
||||
declare ptr @get()
|
||||
declare ptr @foo1()
|
||||
declare ptr @foo2()
|
||||
|
||||
; Don't need to track inline history for non-mutually recursive calls.
|
||||
|
||||
define internal void @x() noinline {
|
||||
ret void
|
||||
}
|
||||
|
||||
define internal void @y() {
|
||||
call void @x()
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @z() {
|
||||
; CHECK-LABEL: define void @z() {
|
||||
; CHECK-NEXT: call void @x(){{$}}
|
||||
; CHECK-NEXT: ret void
|
||||
call void @y()
|
||||
ret void
|
||||
}
|
||||
|
||||
; Indirect calls may be devirtualized, they need inline history tracking. In
|
||||
; this case, @s is the history, but it gets deleted so the inline history
|
||||
; metadata becomes null. This is fine since we cannot infinitely inline through
|
||||
; @s anymore as it doesn't exist.
|
||||
|
||||
define internal void @s() {
|
||||
%p = call ptr @get()
|
||||
call void %p()
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @t() {
|
||||
; CHECK-LABEL: define void @t() {
|
||||
; CHECK-NEXT: %p.i = call ptr @get()
|
||||
; CHECK-NEXT: call void %p.i(), !inline_history [[HISTORY_T:![0-9]+]]
|
||||
; CHECK-NEXT: ret void
|
||||
call void @s()
|
||||
ret void
|
||||
}
|
||||
|
||||
; This will inline @f1 into @a, causing two new calls to @f2, which will get inlined for two calls to @f1.
|
||||
; The inline history should stop recursive inlining here.
|
||||
|
||||
define internal void @f1(ptr %p) {
|
||||
call void %p(ptr @f1)
|
||||
ret void
|
||||
}
|
||||
|
||||
define internal void @f2(ptr %p) {
|
||||
call void %p(ptr @f2)
|
||||
call void %p(ptr @f2)
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @b() {
|
||||
; CHECK-LABEL: define void @b() {
|
||||
; CHECK-NEXT: call void @f1(ptr @f2), !inline_history [[HISTORY_B:![0-9]+]]
|
||||
; CHECK-NEXT: call void @f1(ptr @f2), !inline_history [[HISTORY_B:![0-9]+]]
|
||||
; CHECK-NEXT: ret void
|
||||
;
|
||||
call void @a()
|
||||
ret void
|
||||
}
|
||||
|
||||
define internal void @a() {
|
||||
call void @f1(ptr @f2)
|
||||
ret void
|
||||
}
|
||||
|
||||
; Check that the inline history is
|
||||
; {callee, processed call's inline_history, just-inlined call's inline_history}
|
||||
|
||||
define void @m(ptr %p) noinline {
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @n() {
|
||||
call void @m(ptr @n2), !inline_history !{ptr @foo1}
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @n2() {
|
||||
call void @m(ptr @n)
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @o() {
|
||||
; CHECK-LABEL: define void @o() {
|
||||
; CHECK-NEXT: call void @m(ptr @n2), !inline_history [[HISTORY_O:![0-9]+]]
|
||||
; CHECK-NEXT: ret void
|
||||
call void @n(), !inline_history !{ptr @foo2}
|
||||
ret void
|
||||
}
|
||||
|
||||
; CHECK-DAG: [[HISTORY_T]] = distinct !{null}
|
||||
; CHECK-DAG: [[HISTORY_B]] = !{ptr @f2, ptr @f1}
|
||||
; CHECK-DAG: [[HISTORY_O]] = !{ptr @n, ptr @foo2, ptr @foo1}
|
||||
@ -14,7 +14,7 @@
|
||||
; CHECK: Size after inlining: 17
|
||||
; CHECK: NOT Inlining (cost=never): noinline function attribute, Call: %call_test = tail call float @test(float %fneg, float %common.ret18.op.i)
|
||||
; CHECK: NOT Inlining (cost=never): noinline function attribute, Call: %call_test.i = tail call float @test(float %x, float %call.i)
|
||||
; CHECK: NOT Inlining (cost=never): recursive, Call: %call.i = tail call float @inline_rec_true_successor(float %x, float %scale)
|
||||
; CHECK: Skipping inlining due to history: inline_rec_true_successor -> inline_rec_true_successor
|
||||
; CHECK: Updated inlining SCC: (test, inline_rec_true_successor)
|
||||
|
||||
; CHECK: Inlining calls in: test
|
||||
|
||||
@ -1,55 +0,0 @@
|
||||
; RUN: not opt -passes=verify < %s 2>&1 | FileCheck %s
|
||||
|
||||
@x = global i32 0
|
||||
|
||||
define void @f() {
|
||||
ret void
|
||||
}
|
||||
|
||||
; CHECK: !inline_history should only exist on calls
|
||||
define void @wrong_instr(ptr %x) {
|
||||
load ptr, ptr %x, !inline_history !{ptr @wrong_instr}
|
||||
ret void
|
||||
}
|
||||
|
||||
; CHECK: !inline_history operands must be functions or null
|
||||
define void @global_value_operand() {
|
||||
call void @f(), !inline_history !{ptr @x}
|
||||
ret void
|
||||
}
|
||||
|
||||
; CHECK: !inline_history operands must be functions or null
|
||||
define void @metadata_operand() {
|
||||
call void @f(), !inline_history !{!0}
|
||||
ret void
|
||||
}
|
||||
|
||||
; CHECK: !inline_history operands must be functions or null
|
||||
define void @nullptr_operand() {
|
||||
call void @f(), !inline_history !{ptr null}
|
||||
ret void
|
||||
}
|
||||
|
||||
; CHECK-NOT: !inline_history operands must be functions or null
|
||||
|
||||
define void @empty_metadata() {
|
||||
call void @f(), !inline_history !{}
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @null_metadata() {
|
||||
call void @f(), !inline_history !{null}
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @function_metadata() {
|
||||
call void @f(), !inline_history !{ptr @f}
|
||||
ret void
|
||||
}
|
||||
|
||||
define void @mixed_metadata() {
|
||||
call void @f(), !inline_history !{null, ptr @f, null, ptr @mixed_metadata}
|
||||
ret void
|
||||
}
|
||||
|
||||
!0 = !{}
|
||||
@ -12,29 +12,31 @@ using namespace llvm;
|
||||
|
||||
namespace {
|
||||
|
||||
class NoFooInlineOrder : public InlineOrder {
|
||||
class NoFooInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
|
||||
public:
|
||||
NoFooInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
|
||||
ModuleAnalysisManager &MAM, Module &M) {
|
||||
DefaultInlineOrder = getDefaultInlineOrder(FAM, Params, MAM, M);
|
||||
}
|
||||
size_t size() override { return DefaultInlineOrder->size(); }
|
||||
void push(CallBase *Elt) override {
|
||||
// We ignore callees named "foo"
|
||||
if (Elt->getCalledFunction()->getName() == "foo") {
|
||||
void push(const std::pair<CallBase *, int> &Elt) override {
|
||||
// We ignore calles named "foo"
|
||||
if (Elt.first->getCalledFunction()->getName() == "foo") {
|
||||
DefaultInlineOrder->push(Elt);
|
||||
}
|
||||
}
|
||||
CallBase *pop() override { return DefaultInlineOrder->pop(); }
|
||||
void erase_if(function_ref<bool(CallBase *)> Pred) override {
|
||||
std::pair<CallBase *, int> pop() override {
|
||||
return DefaultInlineOrder->pop();
|
||||
}
|
||||
void erase_if(function_ref<bool(std::pair<CallBase *, int>)> Pred) override {
|
||||
DefaultInlineOrder->erase_if(Pred);
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<InlineOrder> DefaultInlineOrder;
|
||||
std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>> DefaultInlineOrder;
|
||||
};
|
||||
|
||||
std::unique_ptr<InlineOrder>
|
||||
std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
|
||||
NoFooInlineOrderFactory(FunctionAnalysisManager &FAM,
|
||||
const InlineParams &Params, ModuleAnalysisManager &MAM,
|
||||
Module &M) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user