shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/llvm/lib/Transforms/Scalar/GuardWidening.cpp
Jeremy Morse 2425e2940e
[DebugInfo][RemoveDIs] Have getInsertionPtAfterDef return an iterator (#73149) 
Part of the "RemoveDIs" project to remove debug intrinsics requires
passing block-positions around in iterators rather than as instruction
pointers, allowing some debug-info to reside in BasicBlock::iterator.
This means getInsertionPointAfterDef has to return an iterator, and as
it can return no-instruction that means returning an optional iterator.

This patch changes the signature for getInsertionPtAfterDef and then
patches up the various places that use it to handle the different type.
This would overall be an NFC patch, however in
InstCombinerImpl::freezeOtherUses I've started skipping any debug
intrinsics at the returned insert-position. This should not have any
_meaningful_ effect on the compiler output: at worst it means variable
assignments that are skipped will now cover the freeze instruction and
anything inserted before it, which should be inconsequential.

Sadly: this makes the function signature ugly. This is probably the
ugliest piece of fallout for the "RemoveDIs" work, but it serves the
overall purpose of improving compile times and not allowing `-g` to
affect compiler output, so should be worthwhile in the end.
2023-11-30 12:19:57 +00:00

1028 lines
38 KiB
C++
Raw Blame History

//===- GuardWidening.cpp - ---- Guard widening ----------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the guard widening pass. The semantics of the
// @llvm.experimental.guard intrinsic lets LLVM transform it so that it fails
// more often that it did before the transform. This optimization is called
// "widening" and can be used hoist and common runtime checks in situations like
// these:
//
// %cmp0 = 7 u< Length
// call @llvm.experimental.guard(i1 %cmp0) [ "deopt"(...) ]
// call @unknown_side_effects()
// %cmp1 = 9 u< Length
// call @llvm.experimental.guard(i1 %cmp1) [ "deopt"(...) ]
// ...
//
// =>
//
// %cmp0 = 9 u< Length
// call @llvm.experimental.guard(i1 %cmp0) [ "deopt"(...) ]
// call @unknown_side_effects()
// ...
//
// If %cmp0 is false, @llvm.experimental.guard will "deoptimize" back to a
// generic implementation of the same function, which will have the correct
// semantics from that point onward. It is always _legal_ to deoptimize (so
// replacing %cmp0 with false is "correct"), though it may not always be
// profitable to do so.
//
// NB! This pass is a work in progress. It hasn't been tuned to be "production
// ready" yet. It is known to have quadriatic running time and will not scale
// to large numbers of guards
//
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Scalar/GuardWidening.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/GuardUtils.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/PostDominators.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/GuardUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include <functional>
using namespace llvm;
#define DEBUG_TYPE "guard-widening"
STATISTIC(GuardsEliminated, "Number of eliminated guards");
STATISTIC(CondBranchEliminated, "Number of eliminated conditional branches");
STATISTIC(FreezeAdded, "Number of freeze instruction introduced");
static cl::opt<bool>
WidenBranchGuards("guard-widening-widen-branch-guards", cl::Hidden,
cl::desc("Whether or not we should widen guards "
"expressed as branches by widenable conditions"),
cl::init(true));
namespace {
// Get the condition of \p I. It can either be a guard or a conditional branch.
static Value *getCondition(Instruction *I) {
if (IntrinsicInst *GI = dyn_cast<IntrinsicInst>(I)) {
assert(GI->getIntrinsicID() == Intrinsic::experimental_guard &&
"Bad guard intrinsic?");
return GI->getArgOperand(0);
}
Value *Cond, *WC;
BasicBlock *IfTrueBB, *IfFalseBB;
if (parseWidenableBranch(I, Cond, WC, IfTrueBB, IfFalseBB))
return Cond;
return cast<BranchInst>(I)->getCondition();
}
// Set the condition for \p I to \p NewCond. \p I can either be a guard or a
// conditional branch.
static void setCondition(Instruction *I, Value *NewCond) {
if (IntrinsicInst *GI = dyn_cast<IntrinsicInst>(I)) {
assert(GI->getIntrinsicID() == Intrinsic::experimental_guard &&
"Bad guard intrinsic?");
GI->setArgOperand(0, NewCond);
return;
}
cast<BranchInst>(I)->setCondition(NewCond);
}
// Eliminates the guard instruction properly.
static void eliminateGuard(Instruction *GuardInst, MemorySSAUpdater *MSSAU) {
GuardInst->eraseFromParent();
if (MSSAU)
MSSAU->removeMemoryAccess(GuardInst);
++GuardsEliminated;
}
/// Find a point at which the widened condition of \p Guard should be inserted.
/// When it is represented as intrinsic call, we can do it right before the call
/// instruction. However, when we are dealing with widenable branch, we must
/// account for the following situation: widening should not turn a
/// loop-invariant condition into a loop-variant. It means that if
/// widenable.condition() call is invariant (w.r.t. any loop), the new wide
/// condition should stay invariant. Otherwise there can be a miscompile, like
/// the one described at https://github.com/llvm/llvm-project/issues/60234. The
/// safest way to do it is to expand the new condition at WC's block.
static Instruction *findInsertionPointForWideCondition(Instruction *WCOrGuard) {
if (isGuard(WCOrGuard))
return WCOrGuard;
if (auto WC = extractWidenableCondition(WCOrGuard))
return cast<Instruction>(WC);
return nullptr;
}
class GuardWideningImpl {
DominatorTree &DT;
PostDominatorTree *PDT;
LoopInfo &LI;
AssumptionCache &AC;
MemorySSAUpdater *MSSAU;
/// Together, these describe the region of interest. This might be all of
/// the blocks within a function, or only a given loop's blocks and preheader.
DomTreeNode *Root;
std::function<bool(BasicBlock*)> BlockFilter;
/// The set of guards and conditional branches whose conditions have been
/// widened into dominating guards.
SmallVector<Instruction *, 16> EliminatedGuardsAndBranches;
/// The set of guards which have been widened to include conditions to other
/// guards.
DenseSet<Instruction *> WidenedGuards;
/// Try to eliminate instruction \p Instr by widening it into an earlier
/// dominating guard. \p DFSI is the DFS iterator on the dominator tree that
/// is currently visiting the block containing \p Guard, and \p GuardsPerBlock
/// maps BasicBlocks to the set of guards seen in that block.
bool eliminateInstrViaWidening(
Instruction *Instr, const df_iterator<DomTreeNode *> &DFSI,
const DenseMap<BasicBlock *, SmallVector<Instruction *, 8>>
&GuardsPerBlock);
/// Used to keep track of which widening potential is more effective.
enum WideningScore {
/// Don't widen.
WS_IllegalOrNegative,
/// Widening is performance neutral as far as the cycles spent in check
/// conditions goes (but can still help, e.g., code layout, having less
/// deopt state).
WS_Neutral,
/// Widening is profitable.
WS_Positive,
/// Widening is very profitable. Not significantly different from \c
/// WS_Positive, except by the order.
WS_VeryPositive
};
static StringRef scoreTypeToString(WideningScore WS);
/// Compute the score for widening the condition in \p DominatedInstr
/// into \p WideningPoint.
WideningScore computeWideningScore(Instruction *DominatedInstr,
Instruction *ToWiden,
Instruction *WideningPoint,
SmallVectorImpl<Value *> &ChecksToHoist,
SmallVectorImpl<Value *> &ChecksToWiden);
/// Helper to check if \p V can be hoisted to \p InsertPos.
bool canBeHoistedTo(const Value *V, const Instruction *InsertPos) const {
SmallPtrSet<const Instruction *, 8> Visited;
return canBeHoistedTo(V, InsertPos, Visited);
}
bool canBeHoistedTo(const Value *V, const Instruction *InsertPos,
SmallPtrSetImpl<const Instruction *> &Visited) const;
bool canBeHoistedTo(const SmallVectorImpl<Value *> &Checks,
const Instruction *InsertPos) const {
return all_of(Checks,
[&](const Value *V) { return canBeHoistedTo(V, InsertPos); });
}
/// Helper to hoist \p V to \p InsertPos. Guaranteed to succeed if \c
/// canBeHoistedTo returned true.
void makeAvailableAt(Value *V, Instruction *InsertPos) const;
void makeAvailableAt(const SmallVectorImpl<Value *> &Checks,
Instruction *InsertPos) const {
for (Value *V : Checks)
makeAvailableAt(V, InsertPos);
}
/// Common helper used by \c widenGuard and \c isWideningCondProfitable. Try
/// to generate an expression computing the logical AND of \p ChecksToHoist
/// and \p ChecksToWiden. Return true if the expression computing the AND is
/// only as expensive as computing one of the set of expressions. If \p
/// InsertPt is true then actually generate the resulting expression, make it
/// available at \p InsertPt and return it in \p Result (else no change to the
/// IR is made).
std::optional<Value *> mergeChecks(SmallVectorImpl<Value *> &ChecksToHoist,
SmallVectorImpl<Value *> &ChecksToWiden,
Instruction *InsertPt);
/// Generate the logical AND of \p ChecksToHoist and \p OldCondition and make
/// it available at InsertPt
Value *hoistChecks(SmallVectorImpl<Value *> &ChecksToHoist,
Value *OldCondition, Instruction *InsertPt);
/// Adds freeze to Orig and push it as far as possible very aggressively.
/// Also replaces all uses of frozen instruction with frozen version.
Value *freezeAndPush(Value *Orig, Instruction *InsertPt);
/// Represents a range check of the form \c Base + \c Offset u< \c Length,
/// with the constraint that \c Length is not negative. \c CheckInst is the
/// pre-existing instruction in the IR that computes the result of this range
/// check.
class RangeCheck {
const Value *Base;
const ConstantInt *Offset;
const Value *Length;
ICmpInst *CheckInst;
public:
explicit RangeCheck(const Value *Base, const ConstantInt *Offset,
const Value *Length, ICmpInst *CheckInst)
: Base(Base), Offset(Offset), Length(Length), CheckInst(CheckInst) {}
void setBase(const Value *NewBase) { Base = NewBase; }
void setOffset(const ConstantInt *NewOffset) { Offset = NewOffset; }
const Value *getBase() const { return Base; }
const ConstantInt *getOffset() const { return Offset; }
const APInt &getOffsetValue() const { return getOffset()->getValue(); }
const Value *getLength() const { return Length; };
ICmpInst *getCheckInst() const { return CheckInst; }
void print(raw_ostream &OS, bool PrintTypes = false) {
OS << "Base: ";
Base->printAsOperand(OS, PrintTypes);
OS << " Offset: ";
Offset->printAsOperand(OS, PrintTypes);
OS << " Length: ";
Length->printAsOperand(OS, PrintTypes);
}
LLVM_DUMP_METHOD void dump() {
print(dbgs());
dbgs() << "\n";
}
};
/// Parse \p ToParse into a conjunction (logical-and) of range checks; and
/// append them to \p Checks. Returns true on success, may clobber \c Checks
/// on failure.
bool parseRangeChecks(SmallVectorImpl<Value *> &ToParse,
SmallVectorImpl<RangeCheck> &Checks) {
for (auto CheckCond : ToParse) {
if (!parseRangeChecks(CheckCond, Checks))
return false;
}
return true;
}
bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks);
/// Combine the checks in \p Checks into a smaller set of checks and append
/// them into \p CombinedChecks. Return true on success (i.e. all of checks
/// in \p Checks were combined into \p CombinedChecks). Clobbers \p Checks
/// and \p CombinedChecks on success and on failure.
bool combineRangeChecks(SmallVectorImpl<RangeCheck> &Checks,
SmallVectorImpl<RangeCheck> &CombinedChecks) const;
/// Can we compute the logical AND of \p ChecksToHoist and \p ChecksToWiden
/// for the price of computing only one of the set of expressions?
bool isWideningCondProfitable(SmallVectorImpl<Value *> &ChecksToHoist,
SmallVectorImpl<Value *> &ChecksToWiden) {
return mergeChecks(ChecksToHoist, ChecksToWiden, /*InsertPt=*/nullptr)
.has_value();
}
/// Widen \p ChecksToWiden to fail if any of \p ChecksToHoist is false
void widenGuard(SmallVectorImpl<Value *> &ChecksToHoist,
SmallVectorImpl<Value *> &ChecksToWiden,
Instruction *ToWiden) {
Instruction *InsertPt = findInsertionPointForWideCondition(ToWiden);
auto MergedCheck = mergeChecks(ChecksToHoist, ChecksToWiden, InsertPt);
Value *Result = MergedCheck ? *MergedCheck
: hoistChecks(ChecksToHoist,
getCondition(ToWiden), InsertPt);
if (isGuardAsWidenableBranch(ToWiden)) {
setWidenableBranchCond(cast<BranchInst>(ToWiden), Result);
return;
}
setCondition(ToWiden, Result);
}
public:
explicit GuardWideningImpl(DominatorTree &DT, PostDominatorTree *PDT,
LoopInfo &LI, AssumptionCache &AC,
MemorySSAUpdater *MSSAU, DomTreeNode *Root,
std::function<bool(BasicBlock *)> BlockFilter)
: DT(DT), PDT(PDT), LI(LI), AC(AC), MSSAU(MSSAU), Root(Root),
BlockFilter(BlockFilter) {}
/// The entry point for this pass.
bool run();
};
}
static bool isSupportedGuardInstruction(const Instruction *Insn) {
if (isGuard(Insn))
return true;
if (WidenBranchGuards && isGuardAsWidenableBranch(Insn))
return true;
return false;
}
bool GuardWideningImpl::run() {
DenseMap<BasicBlock *, SmallVector<Instruction *, 8>> GuardsInBlock;
bool Changed = false;
for (auto DFI = df_begin(Root), DFE = df_end(Root);
DFI != DFE; ++DFI) {
auto *BB = (*DFI)->getBlock();
if (!BlockFilter(BB))
continue;
auto &CurrentList = GuardsInBlock[BB];
for (auto &I : *BB)
if (isSupportedGuardInstruction(&I))
CurrentList.push_back(cast<Instruction>(&I));
for (auto *II : CurrentList)
Changed |= eliminateInstrViaWidening(II, DFI, GuardsInBlock);
}
assert(EliminatedGuardsAndBranches.empty() || Changed);
for (auto *I : EliminatedGuardsAndBranches)
if (!WidenedGuards.count(I)) {
assert(isa<ConstantInt>(getCondition(I)) && "Should be!");
if (isSupportedGuardInstruction(I))
eliminateGuard(I, MSSAU);
else {
assert(isa<BranchInst>(I) &&
"Eliminated something other than guard or branch?");
++CondBranchEliminated;
}
}
return Changed;
}
bool GuardWideningImpl::eliminateInstrViaWidening(
Instruction *Instr, const df_iterator<DomTreeNode *> &DFSI,
const DenseMap<BasicBlock *, SmallVector<Instruction *, 8>>
&GuardsInBlock) {
SmallVector<Value *> ChecksToHoist;
parseWidenableGuard(Instr, ChecksToHoist);
// Ignore trivial true or false conditions. These instructions will be
// trivially eliminated by any cleanup pass. Do not erase them because other
// guards can possibly be widened into them.
if (ChecksToHoist.empty() ||
(ChecksToHoist.size() == 1 && isa<ConstantInt>(ChecksToHoist.front())))
return false;
Instruction *BestSoFar = nullptr;
auto BestScoreSoFar = WS_IllegalOrNegative;
// In the set of dominating guards, find the one we can merge GuardInst with
// for the most profit.
for (unsigned i = 0, e = DFSI.getPathLength(); i != e; ++i) {
auto *CurBB = DFSI.getPath(i)->getBlock();
if (!BlockFilter(CurBB))
break;
assert(GuardsInBlock.count(CurBB) && "Must have been populated by now!");
const auto &GuardsInCurBB = GuardsInBlock.find(CurBB)->second;
auto I = GuardsInCurBB.begin();
auto E = Instr->getParent() == CurBB ? find(GuardsInCurBB, Instr)
: GuardsInCurBB.end();
#ifndef NDEBUG
{
unsigned Index = 0;
for (auto &I : *CurBB) {
if (Index == GuardsInCurBB.size())
break;
if (GuardsInCurBB[Index] == &I)
Index++;
}
assert(Index == GuardsInCurBB.size() &&
"Guards expected to be in order!");
}
#endif
assert((i == (e - 1)) == (Instr->getParent() == CurBB) && "Bad DFS?");
for (auto *Candidate : make_range(I, E)) {
auto *WideningPoint = findInsertionPointForWideCondition(Candidate);
if (!WideningPoint)
continue;
SmallVector<Value *> CandidateChecks;
parseWidenableGuard(Candidate, CandidateChecks);
auto Score = computeWideningScore(Instr, Candidate, WideningPoint,
ChecksToHoist, CandidateChecks);
LLVM_DEBUG(dbgs() << "Score between " << *Instr << " and " << *Candidate
<< " is " << scoreTypeToString(Score) << "\n");
if (Score > BestScoreSoFar) {
BestScoreSoFar = Score;
BestSoFar = Candidate;
}
}
}
if (BestScoreSoFar == WS_IllegalOrNegative) {
LLVM_DEBUG(dbgs() << "Did not eliminate guard " << *Instr << "\n");
return false;
}
assert(BestSoFar != Instr && "Should have never visited same guard!");
assert(DT.dominates(BestSoFar, Instr) && "Should be!");
LLVM_DEBUG(dbgs() << "Widening " << *Instr << " into " << *BestSoFar
<< " with score " << scoreTypeToString(BestScoreSoFar)
<< "\n");
SmallVector<Value *> ChecksToWiden;
parseWidenableGuard(BestSoFar, ChecksToWiden);
widenGuard(ChecksToHoist, ChecksToWiden, BestSoFar);
auto NewGuardCondition = ConstantInt::getTrue(Instr->getContext());
setCondition(Instr, NewGuardCondition);
EliminatedGuardsAndBranches.push_back(Instr);
WidenedGuards.insert(BestSoFar);
return true;
}
GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore(
Instruction *DominatedInstr, Instruction *ToWiden,
Instruction *WideningPoint, SmallVectorImpl<Value *> &ChecksToHoist,
SmallVectorImpl<Value *> &ChecksToWiden) {
Loop *DominatedInstrLoop = LI.getLoopFor(DominatedInstr->getParent());
Loop *DominatingGuardLoop = LI.getLoopFor(WideningPoint->getParent());
bool HoistingOutOfLoop = false;
if (DominatingGuardLoop != DominatedInstrLoop) {
// Be conservative and don't widen into a sibling loop. TODO: If the
// sibling is colder, we should consider allowing this.
if (DominatingGuardLoop &&
!DominatingGuardLoop->contains(DominatedInstrLoop))
return WS_IllegalOrNegative;
HoistingOutOfLoop = true;
}
if (!canBeHoistedTo(ChecksToHoist, WideningPoint))
return WS_IllegalOrNegative;
// Further in the GuardWideningImpl::hoistChecks the entire condition might be
// widened, not the parsed list of checks. So we need to check the possibility
// of that condition hoisting.
if (!canBeHoistedTo(getCondition(ToWiden), WideningPoint))
return WS_IllegalOrNegative;
// If the guard was conditional executed, it may never be reached
// dynamically. There are two potential downsides to hoisting it out of the
// conditionally executed region: 1) we may spuriously deopt without need and
// 2) we have the extra cost of computing the guard condition in the common
// case. At the moment, we really only consider the second in our heuristic
// here. TODO: evaluate cost model for spurious deopt
// NOTE: As written, this also lets us hoist right over another guard which
// is essentially just another spelling for control flow.
if (isWideningCondProfitable(ChecksToHoist, ChecksToWiden))
return HoistingOutOfLoop ? WS_VeryPositive : WS_Positive;
if (HoistingOutOfLoop)
return WS_Positive;
// For a given basic block \p BB, return its successor which is guaranteed or
// highly likely will be taken as its successor.
auto GetLikelySuccessor = [](const BasicBlock * BB)->const BasicBlock * {
if (auto *UniqueSucc = BB->getUniqueSuccessor())
return UniqueSucc;
auto *Term = BB->getTerminator();
Value *Cond = nullptr;
const BasicBlock *IfTrue = nullptr, *IfFalse = nullptr;
using namespace PatternMatch;
if (!match(Term, m_Br(m_Value(Cond), m_BasicBlock(IfTrue),
m_BasicBlock(IfFalse))))
return nullptr;
// For constant conditions, only one dynamical successor is possible
if (auto *ConstCond = dyn_cast<ConstantInt>(Cond))
return ConstCond->isAllOnesValue() ? IfTrue : IfFalse;
// If one of successors ends with deopt, another one is likely.
if (IfFalse->getPostdominatingDeoptimizeCall())
return IfTrue;
if (IfTrue->getPostdominatingDeoptimizeCall())
return IfFalse;
// TODO: Use branch frequency metatada to allow hoisting through non-deopt
// branches?
return nullptr;
};
// Returns true if we might be hoisting above explicit control flow into a
// considerably hotter block. Note that this completely ignores implicit
// control flow (guards, calls which throw, etc...). That choice appears
// arbitrary (we assume that implicit control flow exits are all rare).
auto MaybeHoistingToHotterBlock = [&]() {
const auto *DominatingBlock = WideningPoint->getParent();
const auto *DominatedBlock = DominatedInstr->getParent();
// Descend as low as we can, always taking the likely successor.
assert(DT.isReachableFromEntry(DominatingBlock) && "Unreached code");
assert(DT.isReachableFromEntry(DominatedBlock) && "Unreached code");
assert(DT.dominates(DominatingBlock, DominatedBlock) && "No dominance");
while (DominatedBlock != DominatingBlock) {
auto *LikelySucc = GetLikelySuccessor(DominatingBlock);
// No likely successor?
if (!LikelySucc)
break;
// Only go down the dominator tree.
if (!DT.properlyDominates(DominatingBlock, LikelySucc))
break;
DominatingBlock = LikelySucc;
}
// Found?
if (DominatedBlock == DominatingBlock)
return false;
// We followed the likely successor chain and went past the dominated
// block. It means that the dominated guard is in dead/very cold code.
if (!DT.dominates(DominatingBlock, DominatedBlock))
return true;
// TODO: diamond, triangle cases
if (!PDT)
return true;
return !PDT->dominates(DominatedBlock, DominatingBlock);
};
return MaybeHoistingToHotterBlock() ? WS_IllegalOrNegative : WS_Neutral;
}
bool GuardWideningImpl::canBeHoistedTo(
const Value *V, const Instruction *Loc,
SmallPtrSetImpl<const Instruction *> &Visited) const {
auto *Inst = dyn_cast<Instruction>(V);
if (!Inst || DT.dominates(Inst, Loc) || Visited.count(Inst))
return true;
if (!isSafeToSpeculativelyExecute(Inst, Loc, &AC, &DT) ||
Inst->mayReadFromMemory())
return false;
Visited.insert(Inst);
// We only want to go _up_ the dominance chain when recursing.
assert(!isa<PHINode>(Loc) &&
"PHIs should return false for isSafeToSpeculativelyExecute");
assert(DT.isReachableFromEntry(Inst->getParent()) &&
"We did a DFS from the block entry!");
return all_of(Inst->operands(),
[&](Value *Op) { return canBeHoistedTo(Op, Loc, Visited); });
}
void GuardWideningImpl::makeAvailableAt(Value *V, Instruction *Loc) const {
auto *Inst = dyn_cast<Instruction>(V);
if (!Inst || DT.dominates(Inst, Loc))
return;
assert(isSafeToSpeculativelyExecute(Inst, Loc, &AC, &DT) &&
!Inst->mayReadFromMemory() &&
"Should've checked with canBeHoistedTo!");
for (Value *Op : Inst->operands())
makeAvailableAt(Op, Loc);
Inst->moveBefore(Loc);
}
// Return Instruction before which we can insert freeze for the value V as close
// to def as possible. If there is no place to add freeze, return empty.
static std::optional<BasicBlock::iterator>
getFreezeInsertPt(Value *V, const DominatorTree &DT) {
auto *I = dyn_cast<Instruction>(V);
if (!I)
return DT.getRoot()->getFirstNonPHIOrDbgOrAlloca()->getIterator();
std::optional<BasicBlock::iterator> Res = I->getInsertionPointAfterDef();
// If there is no place to add freeze - return nullptr.
if (!Res || !DT.dominates(I, &**Res))
return std::nullopt;
Instruction *ResInst = &**Res;
// If there is a User dominated by original I, then it should be dominated
// by Freeze instruction as well.
if (any_of(I->users(), [&](User *U) {
Instruction *User = cast<Instruction>(U);
return ResInst != User && DT.dominates(I, User) &&
!DT.dominates(ResInst, User);
}))
return std::nullopt;
return Res;
}
Value *GuardWideningImpl::freezeAndPush(Value *Orig, Instruction *InsertPt) {
if (isGuaranteedNotToBePoison(Orig, nullptr, InsertPt, &DT))
return Orig;
std::optional<BasicBlock::iterator> InsertPtAtDef =
getFreezeInsertPt(Orig, DT);
if (!InsertPtAtDef) {
FreezeInst *FI = new FreezeInst(Orig, "gw.freeze");
FI->insertBefore(InsertPt);
return FI;
}
if (isa<Constant>(Orig) || isa<GlobalValue>(Orig)) {
BasicBlock::iterator InsertPt = *InsertPtAtDef;
FreezeInst *FI = new FreezeInst(Orig, "gw.freeze");
FI->insertBefore(*InsertPt->getParent(), InsertPt);
return FI;
}
SmallSet<Value *, 16> Visited;
SmallVector<Value *, 16> Worklist;
SmallSet<Instruction *, 16> DropPoisonFlags;
SmallVector<Value *, 16> NeedFreeze;
DenseMap<Value *, FreezeInst *> CacheOfFreezes;
// A bit overloaded data structures. Visited contains constant/GV
// if we already met it. In this case CacheOfFreezes has a freeze if it is
// required.
auto handleConstantOrGlobal = [&](Use &U) {
Value *Def = U.get();
if (!isa<Constant>(Def) && !isa<GlobalValue>(Def))
return false;
if (Visited.insert(Def).second) {
if (isGuaranteedNotToBePoison(Def, nullptr, InsertPt, &DT))
return true;
BasicBlock::iterator InsertPt = *getFreezeInsertPt(Def, DT);
FreezeInst *FI = new FreezeInst(Def, Def->getName() + ".gw.fr");
FI->insertBefore(*InsertPt->getParent(), InsertPt);
CacheOfFreezes[Def] = FI;
}
if (CacheOfFreezes.count(Def))
U.set(CacheOfFreezes[Def]);
return true;
};
Worklist.push_back(Orig);
while (!Worklist.empty()) {
Value *V = Worklist.pop_back_val();
if (!Visited.insert(V).second)
continue;
if (isGuaranteedNotToBePoison(V, nullptr, InsertPt, &DT))
continue;
Instruction *I = dyn_cast<Instruction>(V);
if (!I || canCreateUndefOrPoison(cast<Operator>(I),
/*ConsiderFlagsAndMetadata*/ false)) {
NeedFreeze.push_back(V);
continue;
}
// Check all operands. If for any of them we cannot insert Freeze,
// stop here. Otherwise, iterate.
if (any_of(I->operands(), [&](Value *Op) {
return isa<Instruction>(Op) && !getFreezeInsertPt(Op, DT);
})) {
NeedFreeze.push_back(I);
continue;
}
DropPoisonFlags.insert(I);
for (Use &U : I->operands())
if (!handleConstantOrGlobal(U))
Worklist.push_back(U.get());
}
for (Instruction *I : DropPoisonFlags)
I->dropPoisonGeneratingFlagsAndMetadata();
Value *Result = Orig;
for (Value *V : NeedFreeze) {
BasicBlock::iterator FreezeInsertPt = *getFreezeInsertPt(V, DT);
FreezeInst *FI = new FreezeInst(V, V->getName() + ".gw.fr");
FI->insertBefore(*FreezeInsertPt->getParent(), FreezeInsertPt);
++FreezeAdded;
if (V == Orig)
Result = FI;
V->replaceUsesWithIf(
FI, [&](const Use & U)->bool { return U.getUser() != FI; });
}
return Result;
}
std::optional<Value *>
GuardWideningImpl::mergeChecks(SmallVectorImpl<Value *> &ChecksToHoist,
SmallVectorImpl<Value *> &ChecksToWiden,
Instruction *InsertPt) {
using namespace llvm::PatternMatch;
Value *Result = nullptr;
{
// L >u C0 && L >u C1 -> L >u max(C0, C1)
ConstantInt *RHS0, *RHS1;
Value *LHS;
ICmpInst::Predicate Pred0, Pred1;
// TODO: Support searching for pairs to merge from both whole lists of
// ChecksToHoist and ChecksToWiden.
if (ChecksToWiden.size() == 1 && ChecksToHoist.size() == 1 &&
match(ChecksToWiden.front(),
m_ICmp(Pred0, m_Value(LHS), m_ConstantInt(RHS0))) &&
match(ChecksToHoist.front(),
m_ICmp(Pred1, m_Specific(LHS), m_ConstantInt(RHS1)))) {
ConstantRange CR0 =
ConstantRange::makeExactICmpRegion(Pred0, RHS0->getValue());
ConstantRange CR1 =
ConstantRange::makeExactICmpRegion(Pred1, RHS1->getValue());
// Given what we're doing here and the semantics of guards, it would
// be correct to use a subset intersection, but that may be too
// aggressive in cases we care about.
if (std::optional<ConstantRange> Intersect =
CR0.exactIntersectWith(CR1)) {
APInt NewRHSAP;
CmpInst::Predicate Pred;
if (Intersect->getEquivalentICmp(Pred, NewRHSAP)) {
if (InsertPt) {
ConstantInt *NewRHS =
ConstantInt::get(InsertPt->getContext(), NewRHSAP);
assert(canBeHoistedTo(LHS, InsertPt) && "must be");
makeAvailableAt(LHS, InsertPt);
Result = new ICmpInst(InsertPt, Pred, LHS, NewRHS, "wide.chk");
}
return Result;
}
}
}
}
{
SmallVector<GuardWideningImpl::RangeCheck, 4> Checks, CombinedChecks;
if (parseRangeChecks(ChecksToWiden, Checks) &&
parseRangeChecks(ChecksToHoist, Checks) &&
combineRangeChecks(Checks, CombinedChecks)) {
if (InsertPt) {
for (auto &RC : CombinedChecks) {
makeAvailableAt(RC.getCheckInst(), InsertPt);
if (Result)
Result = BinaryOperator::CreateAnd(RC.getCheckInst(), Result, "",
InsertPt);
else
Result = RC.getCheckInst();
}
assert(Result && "Failed to find result value");
Result->setName("wide.chk");
Result = freezeAndPush(Result, InsertPt);
}
return Result;
}
}
// We were not able to compute ChecksToHoist AND ChecksToWiden for the price
// of one.
return std::nullopt;
}
Value *GuardWideningImpl::hoistChecks(SmallVectorImpl<Value *> &ChecksToHoist,
Value *OldCondition,
Instruction *InsertPt) {
assert(!ChecksToHoist.empty());
IRBuilder<> Builder(InsertPt);
makeAvailableAt(ChecksToHoist, InsertPt);
makeAvailableAt(OldCondition, InsertPt);
Value *Result = Builder.CreateAnd(ChecksToHoist);
Result = freezeAndPush(Result, InsertPt);
Result = Builder.CreateAnd(OldCondition, Result);
Result->setName("wide.chk");
return Result;
}
bool GuardWideningImpl::parseRangeChecks(
Value *CheckCond, SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks) {
using namespace llvm::PatternMatch;
auto *IC = dyn_cast<ICmpInst>(CheckCond);
if (!IC || !IC->getOperand(0)->getType()->isIntegerTy() ||
(IC->getPredicate() != ICmpInst::ICMP_ULT &&
IC->getPredicate() != ICmpInst::ICMP_UGT))
return false;
const Value *CmpLHS = IC->getOperand(0), *CmpRHS = IC->getOperand(1);
if (IC->getPredicate() == ICmpInst::ICMP_UGT)
std::swap(CmpLHS, CmpRHS);
auto &DL = IC->getModule()->getDataLayout();
GuardWideningImpl::RangeCheck Check(
CmpLHS, cast<ConstantInt>(ConstantInt::getNullValue(CmpRHS->getType())),
CmpRHS, IC);
if (!isKnownNonNegative(Check.getLength(), DL))
return false;
// What we have in \c Check now is a correct interpretation of \p CheckCond.
// Try to see if we can move some constant offsets into the \c Offset field.
bool Changed;
auto &Ctx = CheckCond->getContext();
do {
Value *OpLHS;
ConstantInt *OpRHS;
Changed = false;
#ifndef NDEBUG
auto *BaseInst = dyn_cast<Instruction>(Check.getBase());
assert((!BaseInst || DT.isReachableFromEntry(BaseInst->getParent())) &&
"Unreachable instruction?");
#endif
if (match(Check.getBase(), m_Add(m_Value(OpLHS), m_ConstantInt(OpRHS)))) {
Check.setBase(OpLHS);
APInt NewOffset = Check.getOffsetValue() + OpRHS->getValue();
Check.setOffset(ConstantInt::get(Ctx, NewOffset));
Changed = true;
} else if (match(Check.getBase(),
m_Or(m_Value(OpLHS), m_ConstantInt(OpRHS)))) {
KnownBits Known = computeKnownBits(OpLHS, DL);
if ((OpRHS->getValue() & Known.Zero) == OpRHS->getValue()) {
Check.setBase(OpLHS);
APInt NewOffset = Check.getOffsetValue() + OpRHS->getValue();
Check.setOffset(ConstantInt::get(Ctx, NewOffset));
Changed = true;
}
}
} while (Changed);
Checks.push_back(Check);
return true;
}
bool GuardWideningImpl::combineRangeChecks(
SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks,
SmallVectorImpl<GuardWideningImpl::RangeCheck> &RangeChecksOut) const {
unsigned OldCount = Checks.size();
while (!Checks.empty()) {
// Pick all of the range checks with a specific base and length, and try to
// merge them.
const Value *CurrentBase = Checks.front().getBase();
const Value *CurrentLength = Checks.front().getLength();
SmallVector<GuardWideningImpl::RangeCheck, 3> CurrentChecks;
auto IsCurrentCheck = [&](GuardWideningImpl::RangeCheck &RC) {
return RC.getBase() == CurrentBase && RC.getLength() == CurrentLength;
};
copy_if(Checks, std::back_inserter(CurrentChecks), IsCurrentCheck);
erase_if(Checks, IsCurrentCheck);
assert(CurrentChecks.size() != 0 && "We know we have at least one!");
if (CurrentChecks.size() < 3) {
llvm::append_range(RangeChecksOut, CurrentChecks);
continue;
}
// CurrentChecks.size() will typically be 3 here, but so far there has been
// no need to hard-code that fact.
llvm::sort(CurrentChecks, [&](const GuardWideningImpl::RangeCheck &LHS,
const GuardWideningImpl::RangeCheck &RHS) {
return LHS.getOffsetValue().slt(RHS.getOffsetValue());
});
// Note: std::sort should not invalidate the ChecksStart iterator.
const ConstantInt *MinOffset = CurrentChecks.front().getOffset();
const ConstantInt *MaxOffset = CurrentChecks.back().getOffset();
unsigned BitWidth = MaxOffset->getValue().getBitWidth();
if ((MaxOffset->getValue() - MinOffset->getValue())
.ugt(APInt::getSignedMinValue(BitWidth)))
return false;
APInt MaxDiff = MaxOffset->getValue() - MinOffset->getValue();
const APInt &HighOffset = MaxOffset->getValue();
auto OffsetOK = [&](const GuardWideningImpl::RangeCheck &RC) {
return (HighOffset - RC.getOffsetValue()).ult(MaxDiff);
};
if (MaxDiff.isMinValue() || !all_of(drop_begin(CurrentChecks), OffsetOK))
return false;
// We have a series of f+1 checks as:
//
// I+k_0 u< L ... Chk_0
// I+k_1 u< L ... Chk_1
// ...
// I+k_f u< L ... Chk_f
//
// with forall i in [0,f]: k_f-k_i u< k_f-k_0 ... Precond_0
// k_f-k_0 u< INT_MIN+k_f ... Precond_1
// k_f != k_0 ... Precond_2
//
// Claim:
// Chk_0 AND Chk_f implies all the other checks
//
// Informal proof sketch:
//
// We will show that the integer range [I+k_0,I+k_f] does not unsigned-wrap
// (i.e. going from I+k_0 to I+k_f does not cross the -1,0 boundary) and
// thus I+k_f is the greatest unsigned value in that range.
//
// This combined with Ckh_(f+1) shows that everything in that range is u< L.
// Via Precond_0 we know that all of the indices in Chk_0 through Chk_(f+1)
// lie in [I+k_0,I+k_f], this proving our claim.
//
// To see that [I+k_0,I+k_f] is not a wrapping range, note that there are
// two possibilities: I+k_0 u< I+k_f or I+k_0 >u I+k_f (they can't be equal
// since k_0 != k_f). In the former case, [I+k_0,I+k_f] is not a wrapping
// range by definition, and the latter case is impossible:
//
// 0-----I+k_f---I+k_0----L---INT_MAX,INT_MIN------------------(-1)
// xxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
//
// For Chk_0 to succeed, we'd have to have k_f-k_0 (the range highlighted
// with 'x' above) to be at least >u INT_MIN.
RangeChecksOut.emplace_back(CurrentChecks.front());
RangeChecksOut.emplace_back(CurrentChecks.back());
}
assert(RangeChecksOut.size() <= OldCount && "We pessimized!");
return RangeChecksOut.size() != OldCount;
}
#ifndef NDEBUG
StringRef GuardWideningImpl::scoreTypeToString(WideningScore WS) {
switch (WS) {
case WS_IllegalOrNegative:
return "IllegalOrNegative";
case WS_Neutral:
return "Neutral";
case WS_Positive:
return "Positive";
case WS_VeryPositive:
return "VeryPositive";
}
llvm_unreachable("Fully covered switch above!");
}
#endif
PreservedAnalyses GuardWideningPass::run(Function &F,
FunctionAnalysisManager &AM) {
// Avoid requesting analyses if there are no guards or widenable conditions.
auto *GuardDecl = F.getParent()->getFunction(
Intrinsic::getName(Intrinsic::experimental_guard));
bool HasIntrinsicGuards = GuardDecl && !GuardDecl->use_empty();
auto *WCDecl = F.getParent()->getFunction(
Intrinsic::getName(Intrinsic::experimental_widenable_condition));
bool HasWidenableConditions = WCDecl && !WCDecl->use_empty();
if (!HasIntrinsicGuards && !HasWidenableConditions)
return PreservedAnalyses::all();
auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
auto &LI = AM.getResult<LoopAnalysis>(F);
auto &PDT = AM.getResult<PostDominatorTreeAnalysis>(F);
auto &AC = AM.getResult<AssumptionAnalysis>(F);
auto *MSSAA = AM.getCachedResult<MemorySSAAnalysis>(F);
std::unique_ptr<MemorySSAUpdater> MSSAU;
if (MSSAA)
MSSAU = std::make_unique<MemorySSAUpdater>(&MSSAA->getMSSA());
if (!GuardWideningImpl(DT, &PDT, LI, AC, MSSAU ? MSSAU.get() : nullptr,
DT.getRootNode(), [](BasicBlock *) { return true; })
.run())
return PreservedAnalyses::all();
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
PA.preserve<MemorySSAAnalysis>();
return PA;
}
PreservedAnalyses GuardWideningPass::run(Loop &L, LoopAnalysisManager &AM,
LoopStandardAnalysisResults &AR,
LPMUpdater &U) {
BasicBlock *RootBB = L.getLoopPredecessor();
if (!RootBB)
RootBB = L.getHeader();
auto BlockFilter = [&](BasicBlock *BB) {
return BB == RootBB || L.contains(BB);
};
std::unique_ptr<MemorySSAUpdater> MSSAU;
if (AR.MSSA)
MSSAU = std::make_unique<MemorySSAUpdater>(AR.MSSA);
if (!GuardWideningImpl(AR.DT, nullptr, AR.LI, AR.AC,
MSSAU ? MSSAU.get() : nullptr, AR.DT.getNode(RootBB),
BlockFilter)
.run())
return PreservedAnalyses::all();
auto PA = getLoopPassPreservedAnalyses();
if (AR.MSSA)
PA.preserve<MemorySSAAnalysis>();
return PA;
}
Reference in New Issue View Git Blame Copy Permalink