82505fbfc8
(Reland of #190092 with verifier change to look through GlobalAliases) So that it's preserved across all inline invocations rather than just one inliner pass run. This prevents cases where devirtualization in the simplification pipeline uncovers inlining opportunities that should be discarded due to inline history, but we dropped the inline history between inliner pass runs, causing code size to blow up, sometimes exponentially. For compile time reasons, we want to limit this to only call sites that have the potential to inline through SCCs, potentially with the help of devirtualization. This means that the callee is in a non-trivial (Ref)SCC, or the call site was previously an indirect call, which can potentially be devirtualized to call any function. The CGSCCUpdater::InlinedInternalEdges logic still seems to be relevant even with this change, as monster_scc.ll blows up if I remove that code. http://llvm-compile-time-tracker.com/compare.php?from=e830d88e8ae5f44a97cc76136a0a4e83aa9157c0&to=ed535e732fc41b79ab8efda2417886cbd0812f7f&stat=instructions:u Fixes #186926.
284 lines
9.9 KiB
C++
284 lines
9.9 KiB
C++
//===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file implements a custom inliner that handles only functions that
|
|
// are marked as "always inline".
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/Transforms/IPO/AlwaysInliner.h"
|
|
#include "llvm/ADT/SetVector.h"
|
|
#include "llvm/Analysis/AliasAnalysis.h"
|
|
#include "llvm/Analysis/AssumptionCache.h"
|
|
#include "llvm/Analysis/InlineAdvisor.h"
|
|
#include "llvm/Analysis/InlineCost.h"
|
|
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
|
|
#include "llvm/Analysis/ProfileSummaryInfo.h"
|
|
#include "llvm/Analysis/TargetLibraryInfo.h"
|
|
#include "llvm/Analysis/TargetTransformInfo.h"
|
|
#include "llvm/IR/Module.h"
|
|
#include "llvm/InitializePasses.h"
|
|
#include "llvm/Transforms/Utils/Cloning.h"
|
|
#include "llvm/Transforms/Utils/ModuleUtils.h"
|
|
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "inline"
|
|
|
|
namespace {
|
|
|
|
bool AlwaysInlineImpl(
|
|
Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
|
|
FunctionAnalysisManager *FAM,
|
|
function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
|
|
function_ref<AAResults &(Function &)> GetAAR,
|
|
function_ref<TargetTransformInfo &(Function &)> GetTTI,
|
|
function_ref<const TargetLibraryInfo &(Function &)> GetTLI) {
|
|
SmallSetVector<CallBase *, 16> Calls;
|
|
bool Changed = false;
|
|
SmallVector<Function *, 16> InlinedComdatFunctions;
|
|
SmallVector<Function *, 4> NeedFlattening;
|
|
|
|
auto TryInline = [&](CallBase &CB, Function &Callee,
|
|
OptimizationRemarkEmitter &ORE, const char *InlineReason,
|
|
SmallVectorImpl<CallBase *> *NewCallSites =
|
|
nullptr) -> bool {
|
|
Function *Caller = CB.getCaller();
|
|
DebugLoc DLoc = CB.getDebugLoc();
|
|
BasicBlock *Block = CB.getParent();
|
|
|
|
InlineFunctionInfo IFI(GetAssumptionCache, &PSI);
|
|
InlineResult Res = InlineFunction(
|
|
CB, IFI, /*MergeAttributes=*/true, &GetAAR(Callee), InsertLifetime,
|
|
/*TrackInlineHistory=*/NewCallSites != nullptr);
|
|
if (!Res.isSuccess()) {
|
|
ORE.emit([&]() {
|
|
return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
|
|
<< "'" << ore::NV("Callee", &Callee) << "' is not inlined into '"
|
|
<< ore::NV("Caller", Caller)
|
|
<< "': " << ore::NV("Reason", Res.getFailureReason());
|
|
});
|
|
return false;
|
|
}
|
|
|
|
emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee, *Caller,
|
|
InlineCost::getAlways(InlineReason),
|
|
/*ForProfileContext=*/false, DEBUG_TYPE);
|
|
if (FAM)
|
|
FAM->invalidate(*Caller, PreservedAnalyses::none());
|
|
if (NewCallSites)
|
|
*NewCallSites = std::move(IFI.InlinedCallSites);
|
|
return true;
|
|
};
|
|
|
|
for (Function &F : make_early_inc_range(M)) {
|
|
if (F.hasFnAttribute(Attribute::Flatten))
|
|
NeedFlattening.push_back(&F);
|
|
|
|
if (F.isPresplitCoroutine())
|
|
continue;
|
|
|
|
if (F.isDeclaration() || !isInlineViable(F).isSuccess())
|
|
continue;
|
|
|
|
Calls.clear();
|
|
|
|
for (User *U : F.users())
|
|
if (auto *CB = dyn_cast<CallBase>(U))
|
|
if (CB->getCalledFunction() == &F &&
|
|
CB->hasFnAttr(Attribute::AlwaysInline) &&
|
|
!CB->getAttributes().hasFnAttr(Attribute::NoInline))
|
|
Calls.insert(CB);
|
|
|
|
for (CallBase *CB : Calls) {
|
|
OptimizationRemarkEmitter ORE(CB->getCaller());
|
|
Changed |= TryInline(*CB, F, ORE, "always inline attribute");
|
|
}
|
|
|
|
F.removeDeadConstantUsers();
|
|
if (F.hasFnAttribute(Attribute::AlwaysInline) && F.isDefTriviallyDead()) {
|
|
if (F.hasComdat()) {
|
|
InlinedComdatFunctions.push_back(&F);
|
|
} else {
|
|
if (FAM)
|
|
FAM->clear(F, F.getName());
|
|
M.getFunctionList().erase(F);
|
|
Changed = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Flatten functions with the flatten attribute using a local worklist.
|
|
for (Function *F : NeedFlattening) {
|
|
SmallVector<std::pair<CallBase *, int>, 16> Worklist;
|
|
SmallVector<std::pair<Function *, int>, 16> InlineHistory;
|
|
SmallVector<CallBase *> NewCallSites;
|
|
OptimizationRemarkEmitter ORE(F);
|
|
|
|
// Collect initial calls.
|
|
for (BasicBlock &BB : *F) {
|
|
for (Instruction &I : BB) {
|
|
if (auto *CB = dyn_cast<CallBase>(&I)) {
|
|
Function *Callee = CB->getCalledFunction();
|
|
if (!Callee || Callee->isDeclaration())
|
|
continue;
|
|
Worklist.push_back({CB, -1});
|
|
}
|
|
}
|
|
}
|
|
|
|
while (!Worklist.empty()) {
|
|
auto Item = Worklist.pop_back_val();
|
|
CallBase *CB = Item.first;
|
|
int InlineHistoryID = Item.second;
|
|
Function *Callee = CB->getCalledFunction();
|
|
if (!Callee)
|
|
continue;
|
|
|
|
// Detect recursion.
|
|
if (Callee == F) {
|
|
ORE.emit([&]() {
|
|
return OptimizationRemarkMissed("inline", "NotInlined",
|
|
CB->getDebugLoc(), CB->getParent())
|
|
<< "'" << ore::NV("Callee", Callee)
|
|
<< "' is not inlined into '"
|
|
<< ore::NV("Caller", CB->getCaller())
|
|
<< "': recursive call during flattening";
|
|
});
|
|
continue;
|
|
}
|
|
|
|
// Use getAttributeBasedInliningDecision for all attribute-based checks
|
|
// including TTI/TLI compatibility and isInlineViable.
|
|
TargetTransformInfo &CalleeTTI = GetTTI(*Callee);
|
|
auto Decision =
|
|
getAttributeBasedInliningDecision(*CB, Callee, CalleeTTI, GetTLI);
|
|
if (!Decision || !Decision->isSuccess())
|
|
continue;
|
|
|
|
if (!TryInline(*CB, *Callee, ORE, "flatten attribute", &NewCallSites))
|
|
continue;
|
|
|
|
Changed = true;
|
|
|
|
// Add new call sites from the inlined function to the worklist.
|
|
if (!NewCallSites.empty()) {
|
|
int NewHistoryID = InlineHistory.size();
|
|
InlineHistory.push_back({Callee, InlineHistoryID});
|
|
for (CallBase *NewCB : NewCallSites) {
|
|
Function *NewCallee = NewCB->getCalledFunction();
|
|
if (NewCallee && !NewCallee->isDeclaration())
|
|
Worklist.push_back({NewCB, NewHistoryID});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!InlinedComdatFunctions.empty()) {
|
|
// Now we just have the comdat functions. Filter out the ones whose comdats
|
|
// are not actually dead.
|
|
filterDeadComdatFunctions(InlinedComdatFunctions);
|
|
// The remaining functions are actually dead.
|
|
for (Function *F : InlinedComdatFunctions) {
|
|
if (FAM)
|
|
FAM->clear(*F, F->getName());
|
|
M.getFunctionList().erase(F);
|
|
Changed = true;
|
|
}
|
|
}
|
|
|
|
return Changed;
|
|
}
|
|
|
|
struct AlwaysInlinerLegacyPass : public ModulePass {
|
|
bool InsertLifetime;
|
|
|
|
AlwaysInlinerLegacyPass()
|
|
: AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}
|
|
|
|
AlwaysInlinerLegacyPass(bool InsertLifetime)
|
|
: ModulePass(ID), InsertLifetime(InsertLifetime) {}
|
|
|
|
/// Main run interface method.
|
|
bool runOnModule(Module &M) override {
|
|
|
|
auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
|
|
auto GetAAR = [&](Function &F) -> AAResults & {
|
|
return getAnalysis<AAResultsWrapperPass>(F).getAAResults();
|
|
};
|
|
auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
|
|
return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
|
|
};
|
|
auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
|
|
return getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
|
|
};
|
|
auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
|
|
return getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
|
|
};
|
|
|
|
return AlwaysInlineImpl(M, InsertLifetime, PSI, /*FAM=*/nullptr,
|
|
GetAssumptionCache, GetAAR, GetTTI, GetTLI);
|
|
}
|
|
|
|
static char ID; // Pass identification, replacement for typeid
|
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
|
AU.addRequired<AssumptionCacheTracker>();
|
|
AU.addRequired<AAResultsWrapperPass>();
|
|
AU.addRequired<ProfileSummaryInfoWrapperPass>();
|
|
AU.addRequired<TargetLibraryInfoWrapperPass>();
|
|
AU.addRequired<TargetTransformInfoWrapperPass>();
|
|
}
|
|
};
|
|
|
|
} // namespace
|
|
|
|
char AlwaysInlinerLegacyPass::ID = 0;
|
|
INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
|
|
"Inliner for always_inline functions", false, false)
|
|
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
|
|
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
|
|
INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
|
|
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
|
|
INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
|
|
INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
|
|
"Inliner for always_inline functions", false, false)
|
|
|
|
Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
|
|
return new AlwaysInlinerLegacyPass(InsertLifetime);
|
|
}
|
|
|
|
PreservedAnalyses AlwaysInlinerPass::run(Module &M,
|
|
ModuleAnalysisManager &MAM) {
|
|
FunctionAnalysisManager &FAM =
|
|
MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
|
|
auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
|
|
return FAM.getResult<AssumptionAnalysis>(F);
|
|
};
|
|
auto GetAAR = [&](Function &F) -> AAResults & {
|
|
return FAM.getResult<AAManager>(F);
|
|
};
|
|
auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
|
|
return FAM.getResult<TargetIRAnalysis>(F);
|
|
};
|
|
auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
|
|
return FAM.getResult<TargetLibraryAnalysis>(F);
|
|
};
|
|
auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);
|
|
|
|
bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, &FAM,
|
|
GetAssumptionCache, GetAAR, GetTTI, GetTLI);
|
|
if (!Changed)
|
|
return PreservedAnalyses::all();
|
|
|
|
PreservedAnalyses PA;
|
|
// We have already invalidated all analyses on modified functions.
|
|
PA.preserveSet<AllAnalysesOn<Function>>();
|
|
return PA;
|
|
}
|