c8365feed7
This is mostly for test: under contextual profiling, we perform ICP for those indirect callsites which have targets marked as `alwaysinline`. This helped uncover a bug with the way the profile was updated upon ICP, where we were skipping over the update if the target wasn't called in that context. That was resulting in incorrect counts for the indirect BB. Also flyby fix to the total/direct count values, they should be 64-bit (as all counters are in the contextual profile)
325 lines
13 KiB
C++
325 lines
13 KiB
C++
//===- ModuleInliner.cpp - Code related to module inliner -----------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file implements the mechanics required to implement inlining without
|
|
// missing any calls in the module level. It doesn't need any infromation about
|
|
// SCC or call graph, which is different from the SCC inliner. The decisions of
|
|
// which calls are profitable to inline are implemented elsewhere.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/Transforms/IPO/ModuleInliner.h"
|
|
#include "llvm/ADT/ScopeExit.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/ADT/Statistic.h"
|
|
#include "llvm/Analysis/AliasAnalysis.h"
|
|
#include "llvm/Analysis/AssumptionCache.h"
|
|
#include "llvm/Analysis/BlockFrequencyInfo.h"
|
|
#include "llvm/Analysis/CtxProfAnalysis.h"
|
|
#include "llvm/Analysis/InlineAdvisor.h"
|
|
#include "llvm/Analysis/InlineCost.h"
|
|
#include "llvm/Analysis/InlineOrder.h"
|
|
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
|
|
#include "llvm/Analysis/ProfileSummaryInfo.h"
|
|
#include "llvm/Analysis/ReplayInlineAdvisor.h"
|
|
#include "llvm/Analysis/TargetLibraryInfo.h"
|
|
#include "llvm/IR/DiagnosticInfo.h"
|
|
#include "llvm/IR/Function.h"
|
|
#include "llvm/IR/InstIterator.h"
|
|
#include "llvm/IR/Instruction.h"
|
|
#include "llvm/IR/IntrinsicInst.h"
|
|
#include "llvm/IR/Module.h"
|
|
#include "llvm/IR/PassManager.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
#include "llvm/Support/Debug.h"
|
|
#include "llvm/Support/raw_ostream.h"
|
|
#include "llvm/Transforms/Utils/CallPromotionUtils.h"
|
|
#include "llvm/Transforms/Utils/Cloning.h"
|
|
#include <cassert>
|
|
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "module-inline"
|
|
|
|
STATISTIC(NumInlined, "Number of functions inlined");
|
|
STATISTIC(NumDeleted, "Number of functions deleted because all callers found");
|
|
|
|
cl::opt<bool> CtxProfPromoteAlwaysInline(
|
|
"ctx-prof-promote-alwaysinline", cl::init(false), cl::Hidden,
|
|
cl::desc("If using a contextual profile in this module, and an indirect "
|
|
"call target is marked as alwaysinline, perform indirect call "
|
|
"promotion for that target. If multiple targets for an indirect "
|
|
"call site fit this description, they are all promoted."));
|
|
|
|
/// Return true if the specified inline history ID
|
|
/// indicates an inline history that includes the specified function.
|
|
static bool inlineHistoryIncludes(
|
|
Function *F, int InlineHistoryID,
|
|
const SmallVectorImpl<std::pair<Function *, int>> &InlineHistory) {
|
|
while (InlineHistoryID != -1) {
|
|
assert(unsigned(InlineHistoryID) < InlineHistory.size() &&
|
|
"Invalid inline history ID");
|
|
if (InlineHistory[InlineHistoryID].first == F)
|
|
return true;
|
|
InlineHistoryID = InlineHistory[InlineHistoryID].second;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
InlineAdvisor &ModuleInlinerPass::getAdvisor(const ModuleAnalysisManager &MAM,
|
|
FunctionAnalysisManager &FAM,
|
|
Module &M) {
|
|
if (OwnedAdvisor)
|
|
return *OwnedAdvisor;
|
|
|
|
auto *IAA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
|
|
if (!IAA) {
|
|
// It should still be possible to run the inliner as a stand-alone module
|
|
// pass, for test scenarios. In that case, we default to the
|
|
// DefaultInlineAdvisor, which doesn't need to keep state between module
|
|
// pass runs. It also uses just the default InlineParams. In this case, we
|
|
// need to use the provided FAM, which is valid for the duration of the
|
|
// inliner pass, and thus the lifetime of the owned advisor. The one we
|
|
// would get from the MAM can be invalidated as a result of the inliner's
|
|
// activity.
|
|
OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
|
|
M, FAM, Params, InlineContext{LTOPhase, InlinePass::ModuleInliner});
|
|
|
|
return *OwnedAdvisor;
|
|
}
|
|
assert(IAA->getAdvisor() &&
|
|
"Expected a present InlineAdvisorAnalysis also have an "
|
|
"InlineAdvisor initialized");
|
|
return *IAA->getAdvisor();
|
|
}
|
|
|
|
static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI) {
|
|
LibFunc LF;
|
|
|
|
// Either this is a normal library function or a "vectorizable"
|
|
// function. Not using the VFDatabase here because this query
|
|
// is related only to libraries handled via the TLI.
|
|
return TLI.getLibFunc(F, LF) ||
|
|
TLI.isKnownVectorFunctionInLibrary(F.getName());
|
|
}
|
|
|
|
PreservedAnalyses ModuleInlinerPass::run(Module &M,
|
|
ModuleAnalysisManager &MAM) {
|
|
LLVM_DEBUG(dbgs() << "---- Module Inliner is Running ---- \n");
|
|
|
|
auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
|
|
if (!IAA.tryCreate(Params, Mode, {},
|
|
InlineContext{LTOPhase, InlinePass::ModuleInliner})) {
|
|
M.getContext().emitError(
|
|
"Could not setup Inlining Advisor for the requested "
|
|
"mode and/or options");
|
|
return PreservedAnalyses::all();
|
|
}
|
|
|
|
auto &CtxProf = MAM.getResult<CtxProfAnalysis>(M);
|
|
|
|
bool Changed = false;
|
|
|
|
ProfileSummaryInfo *PSI = MAM.getCachedResult<ProfileSummaryAnalysis>(M);
|
|
|
|
FunctionAnalysisManager &FAM =
|
|
MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
|
|
|
|
auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
|
|
return FAM.getResult<TargetLibraryAnalysis>(F);
|
|
};
|
|
|
|
InlineAdvisor &Advisor = getAdvisor(MAM, FAM, M);
|
|
Advisor.onPassEntry();
|
|
|
|
auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(); });
|
|
|
|
// In the module inliner, a priority-based worklist is used for calls across
|
|
// the entire Module. With this module inliner, the inline order is not
|
|
// limited to bottom-up order. More globally scope inline order is enabled.
|
|
// Also, the inline deferral logic become unnecessary in this module inliner.
|
|
// It is possible to use other priority heuristics, e.g. profile-based
|
|
// heuristic.
|
|
//
|
|
// TODO: Here is a huge amount duplicate code between the module inliner and
|
|
// the SCC inliner, which need some refactoring.
|
|
auto Calls = getInlineOrder(FAM, Params, MAM, M);
|
|
assert(Calls != nullptr && "Expected an initialized InlineOrder");
|
|
|
|
// Populate the initial list of calls in this module.
|
|
SetVector<std::pair<CallBase *, Function *>> ICPCandidates;
|
|
for (Function &F : M) {
|
|
auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
|
|
for (Instruction &I : instructions(F)) {
|
|
if (auto *CB = dyn_cast<CallBase>(&I)) {
|
|
if (Function *Callee = CB->getCalledFunction()) {
|
|
if (!Callee->isDeclaration())
|
|
Calls->push({CB, -1});
|
|
else if (!isa<IntrinsicInst>(I)) {
|
|
using namespace ore;
|
|
setInlineRemark(*CB, "unavailable definition");
|
|
ORE.emit([&]() {
|
|
return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I)
|
|
<< NV("Callee", Callee) << " will not be inlined into "
|
|
<< NV("Caller", CB->getCaller())
|
|
<< " because its definition is unavailable"
|
|
<< setIsVerbose();
|
|
});
|
|
}
|
|
} else if (CtxProfPromoteAlwaysInline && CtxProf &&
|
|
CB->isIndirectCall()) {
|
|
CtxProfAnalysis::collectIndirectCallPromotionList(*CB, CtxProf,
|
|
ICPCandidates);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for (auto &[CB, Target] : ICPCandidates) {
|
|
if (auto *DirectCB = promoteCallWithIfThenElse(*CB, *Target, CtxProf))
|
|
Calls->push({DirectCB, -1});
|
|
}
|
|
if (Calls->empty())
|
|
return PreservedAnalyses::all();
|
|
|
|
// When inlining a callee produces new call sites, we want to keep track of
|
|
// the fact that they were inlined from the callee. This allows us to avoid
|
|
// infinite inlining in some obscure cases. To represent this, we use an
|
|
// index into the InlineHistory vector.
|
|
SmallVector<std::pair<Function *, int>, 16> InlineHistory;
|
|
|
|
// Track the dead functions to delete once finished with inlining calls. We
|
|
// defer deleting these to make it easier to handle the call graph updates.
|
|
SmallVector<Function *, 4> DeadFunctions;
|
|
|
|
// Loop forward over all of the calls.
|
|
while (!Calls->empty()) {
|
|
auto P = Calls->pop();
|
|
CallBase *CB = P.first;
|
|
const int InlineHistoryID = P.second;
|
|
Function &F = *CB->getCaller();
|
|
Function &Callee = *CB->getCalledFunction();
|
|
|
|
LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n"
|
|
<< " Function size: " << F.getInstructionCount()
|
|
<< "\n");
|
|
(void)F;
|
|
|
|
auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
|
|
return FAM.getResult<AssumptionAnalysis>(F);
|
|
};
|
|
|
|
if (InlineHistoryID != -1 &&
|
|
inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
|
|
setInlineRemark(*CB, "recursive");
|
|
continue;
|
|
}
|
|
|
|
auto Advice = Advisor.getAdvice(*CB, /*OnlyMandatory*/ false);
|
|
// Check whether we want to inline this callsite.
|
|
if (!Advice->isInliningRecommended()) {
|
|
Advice->recordUnattemptedInlining();
|
|
continue;
|
|
}
|
|
|
|
// Setup the data structure used to plumb customization into the
|
|
// `InlineFunction` routine.
|
|
InlineFunctionInfo IFI(
|
|
GetAssumptionCache, PSI,
|
|
&FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
|
|
&FAM.getResult<BlockFrequencyAnalysis>(Callee));
|
|
|
|
InlineResult IR =
|
|
InlineFunction(*CB, IFI, CtxProf, /*MergeAttributes=*/true,
|
|
&FAM.getResult<AAManager>(*CB->getCaller()));
|
|
if (!IR.isSuccess()) {
|
|
Advice->recordUnsuccessfulInlining(IR);
|
|
continue;
|
|
}
|
|
|
|
Changed = true;
|
|
++NumInlined;
|
|
|
|
LLVM_DEBUG(dbgs() << " Size after inlining: " << F.getInstructionCount()
|
|
<< "\n");
|
|
|
|
// Add any new callsites to defined functions to the worklist.
|
|
if (!IFI.InlinedCallSites.empty()) {
|
|
int NewHistoryID = InlineHistory.size();
|
|
InlineHistory.push_back({&Callee, InlineHistoryID});
|
|
|
|
for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
|
|
Function *NewCallee = ICB->getCalledFunction();
|
|
if (!NewCallee) {
|
|
// Try to promote an indirect (virtual) call without waiting for
|
|
// the post-inline cleanup and the next DevirtSCCRepeatedPass
|
|
// iteration because the next iteration may not happen and we may
|
|
// miss inlining it.
|
|
// FIXME: enable for ctxprof.
|
|
if (!CtxProf)
|
|
if (tryPromoteCall(*ICB))
|
|
NewCallee = ICB->getCalledFunction();
|
|
}
|
|
if (NewCallee)
|
|
if (!NewCallee->isDeclaration())
|
|
Calls->push({ICB, NewHistoryID});
|
|
}
|
|
}
|
|
|
|
// For local functions, check whether this makes the callee trivially
|
|
// dead. In that case, we can drop the body of the function eagerly
|
|
// which may reduce the number of callers of other functions to one,
|
|
// changing inline cost thresholds.
|
|
bool CalleeWasDeleted = false;
|
|
if (Callee.hasLocalLinkage()) {
|
|
// To check this we also need to nuke any dead constant uses (perhaps
|
|
// made dead by this operation on other functions).
|
|
Callee.removeDeadConstantUsers();
|
|
// if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
|
|
if (Callee.use_empty() && !isKnownLibFunction(Callee, GetTLI(Callee))) {
|
|
Calls->erase_if([&](const std::pair<CallBase *, int> &Call) {
|
|
return Call.first->getCaller() == &Callee;
|
|
});
|
|
// Clear the body and queue the function itself for deletion when we
|
|
// finish inlining.
|
|
// Note that after this point, it is an error to do anything other
|
|
// than use the callee's address or delete it.
|
|
Callee.dropAllReferences();
|
|
assert(!is_contained(DeadFunctions, &Callee) &&
|
|
"Cannot put cause a function to become dead twice!");
|
|
DeadFunctions.push_back(&Callee);
|
|
CalleeWasDeleted = true;
|
|
}
|
|
}
|
|
if (CalleeWasDeleted)
|
|
Advice->recordInliningWithCalleeDeleted();
|
|
else
|
|
Advice->recordInlining();
|
|
}
|
|
|
|
// Now that we've finished inlining all of the calls across this module,
|
|
// delete all of the trivially dead functions.
|
|
//
|
|
// Note that this walks a pointer set which has non-deterministic order but
|
|
// that is OK as all we do is delete things and add pointers to unordered
|
|
// sets.
|
|
for (Function *DeadF : DeadFunctions) {
|
|
// Clear out any cached analyses.
|
|
FAM.clear(*DeadF, DeadF->getName());
|
|
|
|
// And delete the actual function from the module.
|
|
M.getFunctionList().erase(DeadF);
|
|
|
|
++NumDeleted;
|
|
}
|
|
|
|
if (!Changed)
|
|
return PreservedAnalyses::all();
|
|
|
|
return PreservedAnalyses::none();
|
|
}
|