shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[LICM & MemorySSA] Make limit flags pass tuning options.

Browse Source 
Summary:
Make the flags in LICM + MemorySSA tuning options in the old and new
pass managers.

Subscribers: mehdi_amini, jlebar, Prazek, george.burgess.iv, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D60490

llvm-svn: 358772
This commit is contained in:
Alina Sbirlea 2019-04-19 17:46:50 +00:00
parent 36c5baef49
commit 43709f7233
9 changed files with 99 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
llvm/include/llvm/Passes/PassBuilder.h
View File

@ -80,6 +80,14 @@ public:
/// Tuning option to enable/disable loop vectorization. Its default value is /// Tuning option to enable/disable loop vectorization. Its default value is
/// that of the flag: `-vectorize-loops`. /// that of the flag: `-vectorize-loops`.
bool LoopVectorization; bool LoopVectorization;
/// Tuning option to cap the number of calls to retrive clobbering accesses in
/// MemorySSA, in LICM.
unsigned LicmMssaOptCap;
/// Tuning option to disable promotion to scalars in LICM with MemorySSA, if
/// the number of access is too large.
unsigned LicmMssaNoAccForPromotionCap;
}; };
/// This class provides access to building LLVM's passes. /// This class provides access to building LLVM's passes.

2
llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h
View File

@ -156,6 +156,8 @@ public:
bool PrepareForThinLTO; bool PrepareForThinLTO;
bool PerformThinLTO; bool PerformThinLTO;
bool DivergentTarget; bool DivergentTarget;
unsigned LicmMssaOptCap;
unsigned LicmMssaNoAccForPromotionCap;
/// Enable profile instrumentation pass. /// Enable profile instrumentation pass.
bool EnablePGOInstrGen; bool EnablePGOInstrGen;

2
llvm/include/llvm/Transforms/Scalar.h
View File

@ -137,6 +137,8 @@ Pass *createIndVarSimplifyPass();
// LICM - This pass is a loop invariant code motion and memory promotion pass. // LICM - This pass is a loop invariant code motion and memory promotion pass.
// //
Pass *createLICMPass(); Pass *createLICMPass();
Pass *createLICMPass(unsigned LicmMssaOptCap,
unsigned LicmMssaNoAccForPromotionCap);
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// //

12
llvm/include/llvm/Transforms/Scalar/LICM.h
View File

@ -38,9 +38,21 @@
namespace llvm { namespace llvm {
extern cl::opt<unsigned> SetLicmMssaOptCap;
extern cl::opt<unsigned> SetLicmMssaNoAccForPromotionCap;
/// Performs Loop Invariant Code Motion Pass. /// Performs Loop Invariant Code Motion Pass.
class LICMPass : public PassInfoMixin<LICMPass> { class LICMPass : public PassInfoMixin<LICMPass> {
unsigned LicmMssaOptCap;
unsigned LicmMssaNoAccForPromotionCap;
public: public:
LICMPass()
: LicmMssaOptCap(SetLicmMssaOptCap),
LicmMssaNoAccForPromotionCap(SetLicmMssaNoAccForPromotionCap) {}
LICMPass(unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap)
: LicmMssaOptCap(LicmMssaOptCap),
LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {}
PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
LoopStandardAnalysisResults &AR, LPMUpdater &U); LoopStandardAnalysisResults &AR, LPMUpdater &U);
}; };

16
llvm/include/llvm/Transforms/Utils/LoopUtils.h
View File

@ -101,6 +101,13 @@ bool formLCSSA(Loop &L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE);
bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI, bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI,
ScalarEvolution *SE); ScalarEvolution *SE);
struct SinkAndHoistLICMFlags {
bool NoOfMemAccTooLarge;
unsigned LicmMssaOptCounter;
unsigned LicmMssaOptCap;
unsigned LicmMssaNoAccForPromotionCap;
};
/// Walk the specified region of the CFG (defined by all blocks /// Walk the specified region of the CFG (defined by all blocks
/// dominated by the specified block, and that are in the current loop) in /// dominated by the specified block, and that are in the current loop) in
/// reverse depth first order w.r.t the DominatorTree. This allows us to visit /// reverse depth first order w.r.t the DominatorTree. This allows us to visit
@ -112,7 +119,7 @@ bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI,
bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
TargetLibraryInfo *, TargetTransformInfo *, Loop *, TargetLibraryInfo *, TargetTransformInfo *, Loop *,
AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *, AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *,
bool, int &, OptimizationRemarkEmitter *); SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *);
/// Walk the specified region of the CFG (defined by all blocks /// Walk the specified region of the CFG (defined by all blocks
/// dominated by the specified block, and that are in the current loop) in depth /// dominated by the specified block, and that are in the current loop) in depth
@ -124,8 +131,8 @@ bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
/// ORE. It returns changed status. /// ORE. It returns changed status.
bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
TargetLibraryInfo *, Loop *, AliasSetTracker *, TargetLibraryInfo *, Loop *, AliasSetTracker *,
MemorySSAUpdater *, ICFLoopSafetyInfo *, bool, int &, MemorySSAUpdater *, ICFLoopSafetyInfo *,
OptimizationRemarkEmitter *); SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *);
/// This function deletes dead loops. The caller of this function needs to /// This function deletes dead loops. The caller of this function needs to
/// guarantee that the loop is infact dead. /// guarantee that the loop is infact dead.
@ -276,8 +283,7 @@ void getLoopAnalysisUsage(AnalysisUsage &AU);
bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT, bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
Loop *CurLoop, AliasSetTracker *CurAST, Loop *CurLoop, AliasSetTracker *CurAST,
MemorySSAUpdater *MSSAU, bool TargetExecutesOncePerLoop, MemorySSAUpdater *MSSAU, bool TargetExecutesOncePerLoop,
bool NoOfMemAccessesTooLarge, SinkAndHoistLICMFlags *LICMFlags = nullptr,
int *LicmMssaOptCap = nullptr,
OptimizationRemarkEmitter *ORE = nullptr); OptimizationRemarkEmitter *ORE = nullptr);
/// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind. /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.

12
llvm/lib/Passes/PassBuilder.cpp
View File

@ -218,6 +218,8 @@ static cl::opt<bool>
PipelineTuningOptions::PipelineTuningOptions() { PipelineTuningOptions::PipelineTuningOptions() {
LoopInterleaving = EnableLoopInterleaving; LoopInterleaving = EnableLoopInterleaving;
LoopVectorization = EnableLoopVectorization; LoopVectorization = EnableLoopVectorization;
LicmMssaOptCap = SetLicmMssaOptCap;
LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
} }
extern cl::opt<bool> EnableHotColdSplit; extern cl::opt<bool> EnableHotColdSplit;
@ -446,7 +448,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
// Rotate Loop - disable header duplication at -Oz // Rotate Loop - disable header duplication at -Oz
LPM1.addPass(LoopRotatePass(Level != Oz)); LPM1.addPass(LoopRotatePass(Level != Oz));
LPM1.addPass(LICMPass()); LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
LPM1.addPass(SimpleLoopUnswitchPass()); LPM1.addPass(SimpleLoopUnswitchPass());
LPM2.addPass(IndVarSimplifyPass()); LPM2.addPass(IndVarSimplifyPass());
LPM2.addPass(LoopIdiomRecognizePass()); LPM2.addPass(LoopIdiomRecognizePass());
@ -506,7 +508,9 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
FPM.addPass(JumpThreadingPass()); FPM.addPass(JumpThreadingPass());
FPM.addPass(CorrelatedValuePropagationPass()); FPM.addPass(CorrelatedValuePropagationPass());
FPM.addPass(DSEPass()); FPM.addPass(DSEPass());
FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging)); FPM.addPass(createFunctionToLoopPassAdaptor(
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
DebugLogging));
for (auto &C : ScalarOptimizerLateEPCallbacks) for (auto &C : ScalarOptimizerLateEPCallbacks)
C(FPM, Level); C(FPM, Level);
@ -907,7 +911,9 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
OptimizePM.addPass(WarnMissedTransformationsPass()); OptimizePM.addPass(WarnMissedTransformationsPass());
OptimizePM.addPass(InstCombinePass()); OptimizePM.addPass(InstCombinePass());
OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>()); OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
OptimizePM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging)); OptimizePM.addPass(createFunctionToLoopPassAdaptor(
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
DebugLogging));
// Now that we've vectorized and unrolled loops, we may have more refined // Now that we've vectorized and unrolled loops, we may have more refined
// alignment information, try to re-derive it here. // alignment information, try to re-derive it here.

15
llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
View File

@ -38,6 +38,7 @@
#include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Scalar/GVN.h" #include "llvm/Transforms/Scalar/GVN.h"
#include "llvm/Transforms/Scalar/InstSimplifyPass.h" #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
#include "llvm/Transforms/Scalar/LICM.h"
#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h" #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
#include "llvm/Transforms/Utils.h" #include "llvm/Transforms/Utils.h"
#include "llvm/Transforms/Vectorize.h" #include "llvm/Transforms/Vectorize.h"
@ -170,6 +171,8 @@ PassManagerBuilder::PassManagerBuilder() {
// the LoopVectorize pass, to be consistent with the new pass manager. // the LoopVectorize pass, to be consistent with the new pass manager.
RerollLoops = RunLoopRerolling; RerollLoops = RunLoopRerolling;
NewGVN = RunNewGVN; NewGVN = RunNewGVN;
LicmMssaOptCap = SetLicmMssaOptCap;
LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
DisableGVNLoadPRE = false; DisableGVNLoadPRE = false;
ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll; ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
VerifyInput = false; VerifyInput = false;
@ -370,7 +373,7 @@ void PassManagerBuilder::addFunctionSimplificationPasses(
} }
// Rotate Loop - disable header duplication at -Oz // Rotate Loop - disable header duplication at -Oz
MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1)); MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
MPM.add(createLICMPass()); // Hoist loop invariants MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
if (EnableSimpleLoopUnswitch) if (EnableSimpleLoopUnswitch)
MPM.add(createSimpleLoopUnswitchLegacyPass()); MPM.add(createSimpleLoopUnswitchLegacyPass());
else else
@ -415,7 +418,7 @@ void PassManagerBuilder::addFunctionSimplificationPasses(
MPM.add(createJumpThreadingPass()); // Thread jumps MPM.add(createJumpThreadingPass()); // Thread jumps
MPM.add(createCorrelatedValuePropagationPass()); MPM.add(createCorrelatedValuePropagationPass());
MPM.add(createDeadStoreEliminationPass()); // Delete dead stores MPM.add(createDeadStoreEliminationPass()); // Delete dead stores
MPM.add(createLICMPass()); MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
addExtensionsToPM(EP_ScalarOptimizerLate, MPM); addExtensionsToPM(EP_ScalarOptimizerLate, MPM);
@ -639,7 +642,7 @@ void PassManagerBuilder::populateModulePassManager(
// later might get benefit of no-alias assumption in clone loop. // later might get benefit of no-alias assumption in clone loop.
if (UseLoopVersioningLICM) { if (UseLoopVersioningLICM) {
MPM.add(createLoopVersioningLICMPass()); // Do LoopVersioningLICM MPM.add(createLoopVersioningLICMPass()); // Do LoopVersioningLICM
MPM.add(createLICMPass()); // Hoist loop invariants MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
} }
// We add a fresh GlobalsModRef run at this point. This is particularly // We add a fresh GlobalsModRef run at this point. This is particularly
@ -696,7 +699,7 @@ void PassManagerBuilder::populateModulePassManager(
MPM.add(createEarlyCSEPass()); MPM.add(createEarlyCSEPass());
MPM.add(createCorrelatedValuePropagationPass()); MPM.add(createCorrelatedValuePropagationPass());
addInstructionCombiningPass(MPM); addInstructionCombiningPass(MPM);
MPM.add(createLICMPass()); MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget)); MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget));
MPM.add(createCFGSimplificationPass()); MPM.add(createCFGSimplificationPass());
addInstructionCombiningPass(MPM); addInstructionCombiningPass(MPM);
@ -738,7 +741,7 @@ void PassManagerBuilder::populateModulePassManager(
// unrolled loop is a inner loop, then the prologue will be inside the // unrolled loop is a inner loop, then the prologue will be inside the
// outer loop. LICM pass can help to promote the runtime check out if the // outer loop. LICM pass can help to promote the runtime check out if the
// checked value is loop invariant. // checked value is loop invariant.
MPM.add(createLICMPass()); MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
} }
MPM.add(createWarnMissedTransformationsPass()); MPM.add(createWarnMissedTransformationsPass());
@ -907,7 +910,7 @@ void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
PM.add(createPostOrderFunctionAttrsLegacyPass()); // Add nocapture. PM.add(createPostOrderFunctionAttrsLegacyPass()); // Add nocapture.
PM.add(createGlobalsAAWrapperPass()); // IP alias analysis. PM.add(createGlobalsAAWrapperPass()); // IP alias analysis.
PM.add(createLICMPass()); // Hoist loop invariants. PM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
PM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds. PM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds.
PM.add(NewGVN ? createNewGVNPass() PM.add(NewGVN ? createNewGVNPass()
: createGVNPass(DisableGVNLoadPRE)); // Remove redundancies. : createGVNPass(DisableGVNLoadPRE)); // Remove redundancies.

78
llvm/lib/Transforms/Scalar/LICM.cpp
View File

@ -115,7 +115,7 @@ LICMN2Theshold("licm-n2-threshold", cl::Hidden, cl::init(0),
// which may not be precise, since optimizeUses is capped. The result is // which may not be precise, since optimizeUses is capped. The result is
// correct, but we may not get as "far up" as possible to get which access is // correct, but we may not get as "far up" as possible to get which access is
// clobbering the one queried. // clobbering the one queried.
static cl::opt<int> LicmMssaOptCap( cl::opt<unsigned> llvm::SetLicmMssaOptCap(
"licm-mssa-optimization-cap", cl::init(100), cl::Hidden, "licm-mssa-optimization-cap", cl::init(100), cl::Hidden,
cl::desc("Enable imprecision in LICM in pathological cases, in exchange " cl::desc("Enable imprecision in LICM in pathological cases, in exchange "
"for faster compile. Caps the MemorySSA clobbering calls.")); "for faster compile. Caps the MemorySSA clobbering calls."));
@ -123,8 +123,8 @@ static cl::opt<int> LicmMssaOptCap(
// Experimentally, memory promotion carries less importance than sinking and // Experimentally, memory promotion carries less importance than sinking and
// hoisting. Limit when we do promotion when using MemorySSA, in order to save // hoisting. Limit when we do promotion when using MemorySSA, in order to save
// compile time. // compile time.
static cl::opt<unsigned> AccessCapForMSSAPromotion( cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
"max-acc-licm-promotion", cl::init(250), cl::Hidden, "licm-mssa-max-acc-promotion", cl::init(250), cl::Hidden,
cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no " cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no "
"effect. When MSSA in LICM is enabled, then this is the maximum " "effect. When MSSA in LICM is enabled, then this is the maximum "
"number of accesses allowed to be present in a loop in order to " "number of accesses allowed to be present in a loop in order to "
@ -151,7 +151,7 @@ static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
AliasAnalysis *AA); AliasAnalysis *AA);
static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU, static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
Loop *CurLoop, Loop *CurLoop,
int &LicmMssaOptCounter); SinkAndHoistLICMFlags &Flags);
static Instruction *CloneInstructionInExitBlock( static Instruction *CloneInstructionInExitBlock(
Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI, Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU); const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);
@ -172,9 +172,15 @@ struct LoopInvariantCodeMotion {
OptimizationRemarkEmitter *ORE, bool DeleteAST); OptimizationRemarkEmitter *ORE, bool DeleteAST);
ASTrackerMapTy &getLoopToAliasSetMap() { return LoopToAliasSetMap; } ASTrackerMapTy &getLoopToAliasSetMap() { return LoopToAliasSetMap; }
LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
unsigned LicmMssaNoAccForPromotionCap)
: LicmMssaOptCap(LicmMssaOptCap),
LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {}
private: private:
ASTrackerMapTy LoopToAliasSetMap; ASTrackerMapTy LoopToAliasSetMap;
unsigned LicmMssaOptCap;
unsigned LicmMssaNoAccForPromotionCap;
std::unique_ptr<AliasSetTracker> std::unique_ptr<AliasSetTracker>
collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AliasAnalysis *AA); collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AliasAnalysis *AA);
@ -185,7 +191,10 @@ private:
struct LegacyLICMPass : public LoopPass { struct LegacyLICMPass : public LoopPass {
static char ID; // Pass identification, replacement for typeid static char ID; // Pass identification, replacement for typeid
LegacyLICMPass() : LoopPass(ID) { LegacyLICMPass(
unsigned LicmMssaOptCap = SetLicmMssaOptCap,
unsigned LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap)
: LoopPass(ID), LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap) {
initializeLegacyLICMPassPass(*PassRegistry::getPassRegistry()); initializeLegacyLICMPassPass(*PassRegistry::getPassRegistry());
} }
@ -267,7 +276,7 @@ PreservedAnalyses LICMPass::run(Loop &L, LoopAnalysisManager &AM,
report_fatal_error("LICM: OptimizationRemarkEmitterAnalysis not " report_fatal_error("LICM: OptimizationRemarkEmitterAnalysis not "
"cached at a higher level"); "cached at a higher level");
LoopInvariantCodeMotion LICM; LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, &AR.TLI, &AR.TTI, &AR.SE, if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, &AR.TLI, &AR.TTI, &AR.SE,
AR.MSSA, ORE, true)) AR.MSSA, ORE, true))
return PreservedAnalyses::all(); return PreservedAnalyses::all();
@ -291,6 +300,10 @@ INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false,
false) false)
Pass *llvm::createLICMPass() { return new LegacyLICMPass(); } Pass *llvm::createLICMPass() { return new LegacyLICMPass(); }
Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
unsigned LicmMssaNoAccForPromotionCap) {
return new LegacyLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
}
/// Hoist expressions out of the specified loop. Note, alias info for inner /// Hoist expressions out of the specified loop. Note, alias info for inner
/// loop is not preserved so it is not a good idea to run LICM multiple /// loop is not preserved so it is not a good idea to run LICM multiple
@ -309,7 +322,7 @@ bool LoopInvariantCodeMotion::runOnLoop(
std::unique_ptr<AliasSetTracker> CurAST; std::unique_ptr<AliasSetTracker> CurAST;
std::unique_ptr<MemorySSAUpdater> MSSAU; std::unique_ptr<MemorySSAUpdater> MSSAU;
bool NoOfMemAccTooLarge = false; bool NoOfMemAccTooLarge = false;
int LicmMssaOptCounter = 0; unsigned LicmMssaOptCounter = 0;
if (!MSSA) { if (!MSSA) {
LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n"); LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n");
@ -324,7 +337,7 @@ bool LoopInvariantCodeMotion::runOnLoop(
for (const auto &MA : *Accesses) { for (const auto &MA : *Accesses) {
(void)MA; (void)MA;
AccessCapCount++; AccessCapCount++;
if (AccessCapCount > AccessCapForMSSAPromotion) { if (AccessCapCount > LicmMssaNoAccForPromotionCap) {
NoOfMemAccTooLarge = true; NoOfMemAccTooLarge = true;
break; break;
} }
@ -351,15 +364,14 @@ bool LoopInvariantCodeMotion::runOnLoop(
// that we are guaranteed to see definitions before we see uses. This allows // that we are guaranteed to see definitions before we see uses. This allows
// us to sink instructions in one pass, without iteration. After sinking // us to sink instructions in one pass, without iteration. After sinking
// instructions, we perform another pass to hoist them out of the loop. // instructions, we perform another pass to hoist them out of the loop.
// SinkAndHoistLICMFlags Flags = {NoOfMemAccTooLarge, LicmMssaOptCounter,
LicmMssaOptCap, LicmMssaNoAccForPromotionCap};
if (L->hasDedicatedExits()) if (L->hasDedicatedExits())
Changed |= sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, TTI, L, Changed |= sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, TTI, L,
CurAST.get(), MSSAU.get(), &SafetyInfo, CurAST.get(), MSSAU.get(), &SafetyInfo, Flags, ORE);
NoOfMemAccTooLarge, LicmMssaOptCounter, ORE);
if (Preheader) if (Preheader)
Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, L, Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, L,
CurAST.get(), MSSAU.get(), &SafetyInfo, CurAST.get(), MSSAU.get(), &SafetyInfo, Flags, ORE);
NoOfMemAccTooLarge, LicmMssaOptCounter, ORE);
// Now that all loop invariants have been removed from the loop, promote any // Now that all loop invariants have been removed from the loop, promote any
// memory references to scalars that we can. // memory references to scalars that we can.
@ -463,8 +475,9 @@ bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
DominatorTree *DT, TargetLibraryInfo *TLI, DominatorTree *DT, TargetLibraryInfo *TLI,
TargetTransformInfo *TTI, Loop *CurLoop, TargetTransformInfo *TTI, Loop *CurLoop,
AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU, AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
ICFLoopSafetyInfo *SafetyInfo, bool NoOfMemAccTooLarge, ICFLoopSafetyInfo *SafetyInfo,
int &LicmMssaOptCounter, OptimizationRemarkEmitter *ORE) { SinkAndHoistLICMFlags &Flags,
OptimizationRemarkEmitter *ORE) {
// Verify inputs. // Verify inputs.
assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
@ -507,8 +520,8 @@ bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
// //
bool FreeInLoop = false; bool FreeInLoop = false;
if (isNotUsedOrFreeInLoop(I, CurLoop, SafetyInfo, TTI, FreeInLoop) && if (isNotUsedOrFreeInLoop(I, CurLoop, SafetyInfo, TTI, FreeInLoop) &&
canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
NoOfMemAccTooLarge, &LicmMssaOptCounter, ORE) && ORE) &&
!I.mayHaveSideEffects()) { !I.mayHaveSideEffects()) {
if (sink(I, LI, DT, CurLoop, SafetyInfo, MSSAU, ORE)) { if (sink(I, LI, DT, CurLoop, SafetyInfo, MSSAU, ORE)) {
if (!FreeInLoop) { if (!FreeInLoop) {
@ -762,8 +775,8 @@ public:
bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI, bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
DominatorTree *DT, TargetLibraryInfo *TLI, Loop *CurLoop, DominatorTree *DT, TargetLibraryInfo *TLI, Loop *CurLoop,
AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU, AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
ICFLoopSafetyInfo *SafetyInfo, bool NoOfMemAccTooLarge, ICFLoopSafetyInfo *SafetyInfo,
int &LicmMssaOptCounter, SinkAndHoistLICMFlags &Flags,
OptimizationRemarkEmitter *ORE) { OptimizationRemarkEmitter *ORE) {
// Verify inputs. // Verify inputs.
assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
@ -816,8 +829,8 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
// and we have accurately duplicated the control flow from the loop header // and we have accurately duplicated the control flow from the loop header
// to that block. // to that block.
if (CurLoop->hasLoopInvariantOperands(&I) && if (CurLoop->hasLoopInvariantOperands(&I) &&
canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
NoOfMemAccTooLarge, &LicmMssaOptCounter, ORE) && ORE) &&
isSafeToExecuteUnconditionally( isSafeToExecuteUnconditionally(
I, DT, CurLoop, SafetyInfo, ORE, I, DT, CurLoop, SafetyInfo, ORE,
CurLoop->getLoopPreheader()->getTerminator())) { CurLoop->getLoopPreheader()->getTerminator())) {
@ -1048,7 +1061,7 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
Loop *CurLoop, AliasSetTracker *CurAST, Loop *CurLoop, AliasSetTracker *CurAST,
MemorySSAUpdater *MSSAU, MemorySSAUpdater *MSSAU,
bool TargetExecutesOncePerLoop, bool TargetExecutesOncePerLoop,
bool NoOfMemAccTooLarge, int *LicmMssaOptCounter, SinkAndHoistLICMFlags *Flags,
OptimizationRemarkEmitter *ORE) { OptimizationRemarkEmitter *ORE) {
// If we don't understand the instruction, bail early. // If we don't understand the instruction, bail early.
if (!isHoistableAndSinkableInst(I)) if (!isHoistableAndSinkableInst(I))
@ -1056,7 +1069,7 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
MemorySSA *MSSA = MSSAU ? MSSAU->getMemorySSA() : nullptr; MemorySSA *MSSA = MSSAU ? MSSAU->getMemorySSA() : nullptr;
if (MSSA) if (MSSA)
assert(LicmMssaOptCounter != nullptr && "Counter cannot be null."); assert(Flags != nullptr && "Flags cannot be null.");
// Loads have extra constraints we have to verify before we can hoist them. // Loads have extra constraints we have to verify before we can hoist them.
if (LoadInst *LI = dyn_cast<LoadInst>(&I)) { if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
@ -1083,8 +1096,7 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
CurLoop, AA); CurLoop, AA);
else else
Invalidated = pointerInvalidatedByLoopWithMSSA( Invalidated = pointerInvalidatedByLoopWithMSSA(
MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop, MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop, *Flags);
*LicmMssaOptCounter);
// Check loop-invariant address because this may also be a sinkable load // Check loop-invariant address because this may also be a sinkable load
// whose address is not necessarily loop-invariant. // whose address is not necessarily loop-invariant.
if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand())) if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
@ -1130,7 +1142,7 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
else else
Invalidated = pointerInvalidatedByLoopWithMSSA( Invalidated = pointerInvalidatedByLoopWithMSSA(
MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop,
*LicmMssaOptCounter); *Flags);
if (Invalidated) if (Invalidated)
return false; return false;
} }
@ -1191,10 +1203,10 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
return true; return true;
// If there are more accesses than the Promotion cap, give up, we're not // If there are more accesses than the Promotion cap, give up, we're not
// walking a list that long. // walking a list that long.
if (NoOfMemAccTooLarge) if (Flags->NoOfMemAccTooLarge)
return false; return false;
// Check store only if there's still "quota" to check clobber. // Check store only if there's still "quota" to check clobber.
if (*LicmMssaOptCounter >= LicmMssaOptCap) if (Flags->LicmMssaOptCounter >= Flags->LicmMssaOptCap)
return false; return false;
// If there are interfering Uses (i.e. their defining access is in the // If there are interfering Uses (i.e. their defining access is in the
// loop), or ordered loads (stored as Defs!), don't move this store. // loop), or ordered loads (stored as Defs!), don't move this store.
@ -1218,7 +1230,7 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
} }
auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI); auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
(*LicmMssaOptCounter)++; Flags->LicmMssaOptCounter++;
// If there are no clobbering Defs in the loop, store is safe to hoist. // If there are no clobbering Defs in the loop, store is safe to hoist.
return MSSA->isLiveOnEntryDef(Source) || return MSSA->isLiveOnEntryDef(Source) ||
!CurLoop->contains(Source->getBlock()); !CurLoop->contains(Source->getBlock());
@ -2236,14 +2248,14 @@ static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU, static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
Loop *CurLoop, Loop *CurLoop,
int &LicmMssaOptCounter) { SinkAndHoistLICMFlags &Flags) {
MemoryAccess *Source; MemoryAccess *Source;
// See declaration of LicmMssaOptCap for usage details. // See declaration of SetLicmMssaOptCap for usage details.
if (LicmMssaOptCounter >= LicmMssaOptCap) if (Flags.LicmMssaOptCounter >= Flags.LicmMssaOptCap)
Source = MU->getDefiningAccess(); Source = MU->getDefiningAccess();
else { else {
Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU); Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
LicmMssaOptCounter++; Flags.LicmMssaOptCounter++;
} }
return !MSSA->isLiveOnEntryDef(Source) && return !MSSA->isLiveOnEntryDef(Source) &&
CurLoop->contains(Source->getBlock()); CurLoop->contains(Source->getBlock());

2
llvm/lib/Transforms/Scalar/LoopSink.cpp
View File

@ -303,7 +303,7 @@ static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI,
// No need to check for instruction's operands are loop invariant. // No need to check for instruction's operands are loop invariant.
assert(L.hasLoopInvariantOperands(I) && assert(L.hasLoopInvariantOperands(I) &&
"Insts in a loop's preheader should have loop invariant operands!"); "Insts in a loop's preheader should have loop invariant operands!");
if (!canSinkOrHoistInst(*I, &AA, &DT, &L, &CurAST, nullptr, false, false)) if (!canSinkOrHoistInst(*I, &AA, &DT, &L, &CurAST, nullptr, false))
continue; continue;
if (sinkInstruction(L, *I, ColdLoopBBs, LoopBlockNumber, LI, DT, BFI)) if (sinkInstruction(L, *I, ColdLoopBBs, LoopBlockNumber, LI, DT, BFI))
Changed = true; Changed = true;