[SCEV] Introduce SCEVUse, use it instead of const SCEV * (NFCI). (#91961)
This patch introduces SCEVUse, which is a tagged pointer containing the used const SCEV *, plus extra bits to store NUW/NSW flags that are only valid at the specific use. This was suggested by @nikic as an alternative to https://github.com/llvm/llvm-project/pull/90742. This patch just updates most SCEV infrastructure to operate on SCEVUse instead of const SCEV *. It does not introduce any code that makes use of the use-specific flags yet which I'll share as follow-ups. Compile-time impact: https://llvm-compile-time-tracker.com/compare.php?from=ee34eb6edccdebc2a752ffecdde5faae6b0d5593&to=5a7727d7819414d2acbc5b6ab740f0fc2363e842&stat=instructions%3Au
This commit is contained in:
Florian Hahn
GitHub
parent
c272252bbd
commit
f8734a5e10
@ -66,6 +66,76 @@ enum SCEVTypes : unsigned short;
|
||||
|
||||
LLVM_ABI extern bool VerifySCEV;
|
||||
|
||||
class SCEV;
|
||||
|
||||
struct SCEVUse : PointerIntPair<const SCEV *, 2> {
|
||||
SCEVUse() : PointerIntPair() { setFromOpaqueValue(nullptr); }
|
||||
SCEVUse(const SCEV *S) : PointerIntPair() { setFromOpaqueValue((void *)S); }
|
||||
SCEVUse(const SCEV *S, unsigned Flags) : PointerIntPair(S, Flags) {}
|
||||
|
||||
operator const SCEV *() const { return getPointer(); }
|
||||
const SCEV *operator->() const { return getPointer(); }
|
||||
|
||||
void *getRawPointer() const { return getOpaqueValue(); }
|
||||
|
||||
unsigned getFlags() const { return getInt(); }
|
||||
|
||||
bool operator==(const SCEVUse &RHS) const {
|
||||
return getRawPointer() == RHS.getRawPointer();
|
||||
}
|
||||
|
||||
bool operator==(const SCEV *RHS) const { return getRawPointer() == RHS; }
|
||||
|
||||
/// Print out the internal representation of this scalar to the specified
|
||||
/// stream. This should really only be used for debugging purposes.
|
||||
void print(raw_ostream &OS) const;
|
||||
|
||||
/// This method is used for debugging.
|
||||
void dump() const;
|
||||
};
|
||||
|
||||
/// Provide PointerLikeTypeTraits for SCEVUse, so it can be used with
|
||||
/// SmallPtrSet, among others.
|
||||
template <> struct PointerLikeTypeTraits<SCEVUse> {
|
||||
static inline void *getAsVoidPointer(SCEVUse U) { return U.getOpaqueValue(); }
|
||||
static inline SCEVUse getFromVoidPointer(void *P) {
|
||||
SCEVUse U;
|
||||
U.setFromOpaqueValue(P);
|
||||
return U;
|
||||
}
|
||||
|
||||
/// The Low bits are used by the PointerIntPair.
|
||||
static constexpr int NumLowBitsAvailable = 0;
|
||||
};
|
||||
|
||||
template <> struct DenseMapInfo<SCEVUse> {
|
||||
static inline SCEVUse getEmptyKey() {
|
||||
uintptr_t Val = static_cast<uintptr_t>(-1);
|
||||
return PointerLikeTypeTraits<SCEVUse>::getFromVoidPointer((void *)Val);
|
||||
}
|
||||
|
||||
static inline SCEVUse getTombstoneKey() {
|
||||
uintptr_t Val = static_cast<uintptr_t>(-2);
|
||||
return PointerLikeTypeTraits<SCEVUse>::getFromVoidPointer((void *)Val);
|
||||
}
|
||||
|
||||
static unsigned getHashValue(SCEVUse U) {
|
||||
return hash_value(U.getRawPointer());
|
||||
}
|
||||
|
||||
static bool isEqual(const SCEVUse LHS, const SCEVUse RHS) {
|
||||
return LHS.getRawPointer() == RHS.getRawPointer();
|
||||
}
|
||||
};
|
||||
|
||||
template <> struct simplify_type<SCEVUse> {
|
||||
using SimpleType = const SCEV *;
|
||||
|
||||
static SimpleType getSimplifiedValue(SCEVUse &Val) {
|
||||
return Val.getPointer();
|
||||
}
|
||||
};
|
||||
|
||||
/// This class represents an analyzed expression in the program. These are
|
||||
/// opaque objects that the client is not allowed to do much with directly.
|
||||
///
|
||||
@ -144,7 +214,7 @@ public:
|
||||
LLVM_ABI Type *getType() const;
|
||||
|
||||
/// Return operands of this SCEV expression.
|
||||
LLVM_ABI ArrayRef<const SCEV *> operands() const;
|
||||
LLVM_ABI ArrayRef<SCEVUse> operands() const;
|
||||
|
||||
/// Return true if the expression is a constant zero.
|
||||
LLVM_ABI bool isZero() const;
|
||||
@ -554,6 +624,7 @@ public:
|
||||
|
||||
/// Notify this ScalarEvolution that \p User directly uses SCEVs in \p Ops.
|
||||
LLVM_ABI void registerUser(const SCEV *User, ArrayRef<const SCEV *> Ops);
|
||||
LLVM_ABI void registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops);
|
||||
|
||||
/// Return true if the SCEV expression contains an undef value.
|
||||
LLVM_ABI bool containsUndefs(const SCEV *S) const;
|
||||
@ -592,46 +663,47 @@ public:
|
||||
unsigned Depth = 0);
|
||||
LLVM_ABI const SCEV *getCastExpr(SCEVTypes Kind, const SCEV *Op, Type *Ty);
|
||||
LLVM_ABI const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty);
|
||||
LLVM_ABI const SCEV *getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
|
||||
|
||||
LLVM_ABI const SCEV *getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
|
||||
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
|
||||
unsigned Depth = 0);
|
||||
const SCEV *getAddExpr(const SCEV *LHS, const SCEV *RHS,
|
||||
const SCEV *getAddExpr(SCEVUse LHS, SCEVUse RHS,
|
||||
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
|
||||
unsigned Depth = 0) {
|
||||
SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
|
||||
SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
|
||||
return getAddExpr(Ops, Flags, Depth);
|
||||
}
|
||||
const SCEV *getAddExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2,
|
||||
const SCEV *getAddExpr(SCEVUse Op0, SCEVUse Op1, SCEVUse Op2,
|
||||
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
|
||||
unsigned Depth = 0) {
|
||||
SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2};
|
||||
SmallVector<SCEVUse, 3> Ops = {Op0, Op1, Op2};
|
||||
return getAddExpr(Ops, Flags, Depth);
|
||||
}
|
||||
LLVM_ABI const SCEV *getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
|
||||
LLVM_ABI const SCEV *getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
|
||||
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
|
||||
unsigned Depth = 0);
|
||||
const SCEV *getMulExpr(const SCEV *LHS, const SCEV *RHS,
|
||||
const SCEV *getMulExpr(SCEVUse LHS, SCEVUse RHS,
|
||||
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
|
||||
unsigned Depth = 0) {
|
||||
SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
|
||||
SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
|
||||
return getMulExpr(Ops, Flags, Depth);
|
||||
}
|
||||
const SCEV *getMulExpr(const SCEV *Op0, const SCEV *Op1, const SCEV *Op2,
|
||||
const SCEV *getMulExpr(SCEVUse Op0, SCEVUse Op1, SCEVUse Op2,
|
||||
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
|
||||
unsigned Depth = 0) {
|
||||
SmallVector<const SCEV *, 3> Ops = {Op0, Op1, Op2};
|
||||
SmallVector<SCEVUse, 3> Ops = {Op0, Op1, Op2};
|
||||
return getMulExpr(Ops, Flags, Depth);
|
||||
}
|
||||
LLVM_ABI const SCEV *getUDivExpr(const SCEV *LHS, const SCEV *RHS);
|
||||
LLVM_ABI const SCEV *getUDivExactExpr(const SCEV *LHS, const SCEV *RHS);
|
||||
LLVM_ABI const SCEV *getURemExpr(const SCEV *LHS, const SCEV *RHS);
|
||||
LLVM_ABI const SCEV *getAddRecExpr(const SCEV *Start, const SCEV *Step,
|
||||
LLVM_ABI const SCEV *getUDivExpr(SCEVUse LHS, SCEVUse RHS);
|
||||
LLVM_ABI const SCEV *getUDivExactExpr(SCEVUse LHS, SCEVUse RHS);
|
||||
LLVM_ABI const SCEV *getURemExpr(SCEVUse LHS, SCEVUse RHS);
|
||||
LLVM_ABI const SCEV *getAddRecExpr(SCEVUse Start, SCEVUse Step, const Loop *L,
|
||||
SCEV::NoWrapFlags Flags);
|
||||
LLVM_ABI const SCEV *getAddRecExpr(SmallVectorImpl<SCEVUse> &Operands,
|
||||
const Loop *L, SCEV::NoWrapFlags Flags);
|
||||
LLVM_ABI const SCEV *getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
|
||||
const Loop *L, SCEV::NoWrapFlags Flags);
|
||||
const SCEV *getAddRecExpr(const SmallVectorImpl<const SCEV *> &Operands,
|
||||
const SCEV *getAddRecExpr(const SmallVectorImpl<SCEVUse> &Operands,
|
||||
const Loop *L, SCEV::NoWrapFlags Flags) {
|
||||
SmallVector<const SCEV *, 4> NewOp(Operands.begin(), Operands.end());
|
||||
SmallVector<SCEVUse, 4> NewOp(Operands.begin(), Operands.end());
|
||||
return getAddRecExpr(NewOp, L, Flags);
|
||||
}
|
||||
|
||||
@ -649,26 +721,25 @@ public:
|
||||
/// instead we use IndexExprs.
|
||||
/// \p IndexExprs The expressions for the indices.
|
||||
LLVM_ABI const SCEV *getGEPExpr(GEPOperator *GEP,
|
||||
ArrayRef<const SCEV *> IndexExprs);
|
||||
LLVM_ABI const SCEV *getGEPExpr(const SCEV *BaseExpr,
|
||||
ArrayRef<const SCEV *> IndexExprs,
|
||||
ArrayRef<SCEVUse> IndexExprs);
|
||||
LLVM_ABI const SCEV *getGEPExpr(SCEVUse BaseExpr,
|
||||
ArrayRef<SCEVUse> IndexExprs,
|
||||
Type *SrcElementTy,
|
||||
GEPNoWrapFlags NW = GEPNoWrapFlags::none());
|
||||
LLVM_ABI const SCEV *getAbsExpr(const SCEV *Op, bool IsNSW);
|
||||
LLVM_ABI const SCEV *getMinMaxExpr(SCEVTypes Kind,
|
||||
SmallVectorImpl<const SCEV *> &Operands);
|
||||
SmallVectorImpl<SCEVUse> &Operands);
|
||||
LLVM_ABI const SCEV *
|
||||
getSequentialMinMaxExpr(SCEVTypes Kind,
|
||||
SmallVectorImpl<const SCEV *> &Operands);
|
||||
LLVM_ABI const SCEV *getSMaxExpr(const SCEV *LHS, const SCEV *RHS);
|
||||
LLVM_ABI const SCEV *getSMaxExpr(SmallVectorImpl<const SCEV *> &Operands);
|
||||
LLVM_ABI const SCEV *getUMaxExpr(const SCEV *LHS, const SCEV *RHS);
|
||||
LLVM_ABI const SCEV *getUMaxExpr(SmallVectorImpl<const SCEV *> &Operands);
|
||||
LLVM_ABI const SCEV *getSMinExpr(const SCEV *LHS, const SCEV *RHS);
|
||||
LLVM_ABI const SCEV *getSMinExpr(SmallVectorImpl<const SCEV *> &Operands);
|
||||
LLVM_ABI const SCEV *getUMinExpr(const SCEV *LHS, const SCEV *RHS,
|
||||
getSequentialMinMaxExpr(SCEVTypes Kind, SmallVectorImpl<SCEVUse> &Operands);
|
||||
LLVM_ABI const SCEV *getSMaxExpr(SCEVUse LHS, SCEVUse RHS);
|
||||
LLVM_ABI const SCEV *getSMaxExpr(SmallVectorImpl<SCEVUse> &Operands);
|
||||
LLVM_ABI const SCEV *getUMaxExpr(SCEVUse LHS, SCEVUse RHS);
|
||||
LLVM_ABI const SCEV *getUMaxExpr(SmallVectorImpl<SCEVUse> &Operands);
|
||||
LLVM_ABI const SCEV *getSMinExpr(SCEVUse LHS, SCEVUse RHS);
|
||||
LLVM_ABI const SCEV *getSMinExpr(SmallVectorImpl<SCEVUse> &Operands);
|
||||
LLVM_ABI const SCEV *getUMinExpr(SCEVUse LHS, SCEVUse RHS,
|
||||
bool Sequential = false);
|
||||
LLVM_ABI const SCEV *getUMinExpr(SmallVectorImpl<const SCEV *> &Operands,
|
||||
LLVM_ABI const SCEV *getUMinExpr(SmallVectorImpl<SCEVUse> &Operands,
|
||||
bool Sequential = false);
|
||||
LLVM_ABI const SCEV *getUnknown(Value *V);
|
||||
LLVM_ABI const SCEV *getCouldNotCompute();
|
||||
@ -717,7 +788,7 @@ public:
|
||||
/// To compute the difference between two unrelated pointers, you can
|
||||
/// explicitly convert the arguments using getPtrToIntExpr(), for pointer
|
||||
/// types that support it.
|
||||
LLVM_ABI const SCEV *getMinusSCEV(const SCEV *LHS, const SCEV *RHS,
|
||||
LLVM_ABI const SCEV *getMinusSCEV(SCEVUse LHS, SCEVUse RHS,
|
||||
SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap,
|
||||
unsigned Depth = 0);
|
||||
|
||||
@ -773,9 +844,8 @@ public:
|
||||
|
||||
/// Promote the operands to the wider of the types using zero-extension, and
|
||||
/// then perform a umin operation with them. N-ary function.
|
||||
LLVM_ABI const SCEV *
|
||||
getUMinFromMismatchedTypes(SmallVectorImpl<const SCEV *> &Ops,
|
||||
bool Sequential = false);
|
||||
LLVM_ABI const SCEV *getUMinFromMismatchedTypes(SmallVectorImpl<SCEVUse> &Ops,
|
||||
bool Sequential = false);
|
||||
|
||||
/// Transitively follow the chain of pointer-type operands until reaching a
|
||||
/// SCEV that does not have a single pointer operand. This returns a
|
||||
@ -2177,7 +2247,7 @@ private:
|
||||
void forgetBackedgeTakenCounts(const Loop *L, bool Predicated);
|
||||
|
||||
/// Drop memoized information for all \p SCEVs.
|
||||
void forgetMemoizedResults(ArrayRef<const SCEV *> SCEVs);
|
||||
void forgetMemoizedResults(ArrayRef<SCEVUse> SCEVs);
|
||||
|
||||
/// Helper for forgetMemoizedResults.
|
||||
void forgetMemoizedResultsImpl(const SCEV *S);
|
||||
@ -2186,7 +2256,7 @@ private:
|
||||
/// from ValueExprMap and collect SCEV expressions in \p ToForget
|
||||
void visitAndClearUsers(SmallVectorImpl<Instruction *> &Worklist,
|
||||
SmallPtrSetImpl<Instruction *> &Visited,
|
||||
SmallVectorImpl<const SCEV *> &ToForget);
|
||||
SmallVectorImpl<SCEVUse> &ToForget);
|
||||
|
||||
/// Erase Value from ValueExprMap and ExprValueMap.
|
||||
void eraseValueFromMap(Value *V);
|
||||
@ -2236,12 +2306,12 @@ private:
|
||||
/// Return a scope which provides an upper bound on the defining scope for
|
||||
/// a SCEV with the operands in Ops. The outparam Precise is set if the
|
||||
/// bound found is a precise bound (i.e. must be the defining scope.)
|
||||
const Instruction *getDefiningScopeBound(ArrayRef<const SCEV *> Ops,
|
||||
const Instruction *getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
|
||||
bool &Precise);
|
||||
|
||||
/// Wrapper around the above for cases which don't care if the bound
|
||||
/// is precise.
|
||||
const Instruction *getDefiningScopeBound(ArrayRef<const SCEV *> Ops);
|
||||
const Instruction *getDefiningScopeBound(ArrayRef<SCEVUse> Ops);
|
||||
|
||||
/// Given two instructions in the same function, return true if we can
|
||||
/// prove B must execute given A executes.
|
||||
@ -2315,16 +2385,16 @@ private:
|
||||
bool canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride, bool IsSigned);
|
||||
|
||||
/// Get add expr already created or create a new one.
|
||||
const SCEV *getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
|
||||
const SCEV *getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
|
||||
SCEV::NoWrapFlags Flags);
|
||||
|
||||
/// Get mul expr already created or create a new one.
|
||||
const SCEV *getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
|
||||
const SCEV *getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
|
||||
SCEV::NoWrapFlags Flags);
|
||||
|
||||
// Get addrec expr already created or create a new one.
|
||||
const SCEV *getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
|
||||
const Loop *L, SCEV::NoWrapFlags Flags);
|
||||
const SCEV *getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops, const Loop *L,
|
||||
SCEV::NoWrapFlags Flags);
|
||||
|
||||
/// Return x if \p Val is f(x) where f is a 1-1 function.
|
||||
const SCEV *stripInjectiveFunctions(const SCEV *Val) const;
|
||||
@ -2337,6 +2407,7 @@ private:
|
||||
/// Look for a SCEV expression with type `SCEVType` and operands `Ops` in
|
||||
/// `UniqueSCEVs`. Return if found, else nullptr.
|
||||
SCEV *findExistingSCEVInCache(SCEVTypes SCEVType, ArrayRef<const SCEV *> Ops);
|
||||
SCEV *findExistingSCEVInCache(SCEVTypes SCEVType, ArrayRef<SCEVUse> Ops);
|
||||
|
||||
/// Get reachable blocks in this function, making limited use of SCEV
|
||||
/// reasoning about conditions.
|
||||
@ -2345,8 +2416,7 @@ private:
|
||||
|
||||
/// Return the given SCEV expression with a new set of operands.
|
||||
/// This preserves the origial nowrap flags.
|
||||
const SCEV *getWithOperands(const SCEV *S,
|
||||
SmallVectorImpl<const SCEV *> &NewOps);
|
||||
const SCEV *getWithOperands(const SCEV *S, SmallVectorImpl<SCEVUse> &NewOps);
|
||||
|
||||
FoldingSet<SCEV> UniqueSCEVs;
|
||||
FoldingSet<SCEVPredicate> UniquePreds;
|
||||
|
||||
@ -94,9 +94,9 @@ public:
|
||||
static bool classof(const SCEV *S) { return S->getSCEVType() == scVScale; }
|
||||
};
|
||||
|
||||
inline unsigned short computeExpressionSize(ArrayRef<const SCEV *> Args) {
|
||||
inline unsigned short computeExpressionSize(ArrayRef<SCEVUse> Args) {
|
||||
APInt Size(16, 1);
|
||||
for (const auto *Arg : Args)
|
||||
for (const SCEV *Arg : Args)
|
||||
Size = Size.uadd_sat(APInt(16, Arg->getExpressionSize()));
|
||||
return (unsigned short)Size.getZExtValue();
|
||||
}
|
||||
@ -104,19 +104,19 @@ inline unsigned short computeExpressionSize(ArrayRef<const SCEV *> Args) {
|
||||
/// This is the base class for unary cast operator classes.
|
||||
class SCEVCastExpr : public SCEV {
|
||||
protected:
|
||||
const SCEV *Op;
|
||||
SCEVUse Op;
|
||||
Type *Ty;
|
||||
|
||||
LLVM_ABI SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
|
||||
const SCEV *op, Type *ty);
|
||||
SCEVUse op, Type *ty);
|
||||
|
||||
public:
|
||||
const SCEV *getOperand() const { return Op; }
|
||||
const SCEV *getOperand(unsigned i) const {
|
||||
SCEVUse getOperand() const { return Op; }
|
||||
SCEVUse getOperand(unsigned i) const {
|
||||
assert(i == 0 && "Operand index out of range!");
|
||||
return Op;
|
||||
}
|
||||
ArrayRef<const SCEV *> operands() const { return Op; }
|
||||
ArrayRef<SCEVUse> operands() const { return Op; }
|
||||
size_t getNumOperands() const { return 1; }
|
||||
Type *getType() const { return Ty; }
|
||||
|
||||
@ -133,11 +133,12 @@ public:
|
||||
class SCEVPtrToIntExpr : public SCEVCastExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op, Type *ITy);
|
||||
SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, SCEVUse Op, Type *ITy);
|
||||
|
||||
public:
|
||||
/// Methods for support type inquiry through isa, cast, and dyn_cast:
|
||||
static bool classof(const SCEV *S) { return S->getSCEVType() == scPtrToInt; }
|
||||
static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
|
||||
};
|
||||
|
||||
/// This class represents a cast from a pointer to a pointer-sized integer
|
||||
@ -156,7 +157,7 @@ public:
|
||||
class SCEVIntegralCastExpr : public SCEVCastExpr {
|
||||
protected:
|
||||
LLVM_ABI SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
|
||||
const SCEV *op, Type *ty);
|
||||
SCEVUse op, Type *ty);
|
||||
|
||||
public:
|
||||
/// Methods for support type inquiry through isa, cast, and dyn_cast:
|
||||
@ -171,7 +172,7 @@ public:
|
||||
class SCEVTruncateExpr : public SCEVIntegralCastExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVTruncateExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty);
|
||||
SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op, Type *ty);
|
||||
|
||||
public:
|
||||
/// Methods for support type inquiry through isa, cast, and dyn_cast:
|
||||
@ -183,7 +184,7 @@ public:
|
||||
class SCEVZeroExtendExpr : public SCEVIntegralCastExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty);
|
||||
SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op, Type *ty);
|
||||
|
||||
public:
|
||||
/// Methods for support type inquiry through isa, cast, and dyn_cast:
|
||||
@ -197,7 +198,7 @@ public:
|
||||
class SCEVSignExtendExpr : public SCEVIntegralCastExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty);
|
||||
SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op, Type *ty);
|
||||
|
||||
public:
|
||||
/// Methods for support type inquiry through isa, cast, and dyn_cast:
|
||||
@ -214,25 +215,23 @@ protected:
|
||||
// arrays with its SCEVAllocator, so this class just needs a simple
|
||||
// pointer rather than a more elaborate vector-like data structure.
|
||||
// This also avoids the need for a non-trivial destructor.
|
||||
const SCEV *const *Operands;
|
||||
const SCEVUse *Operands;
|
||||
size_t NumOperands;
|
||||
|
||||
SCEVNAryExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
|
||||
const SCEV *const *O, size_t N)
|
||||
SCEVNAryExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, const SCEVUse *O,
|
||||
size_t N)
|
||||
: SCEV(ID, T, computeExpressionSize(ArrayRef(O, N))), Operands(O),
|
||||
NumOperands(N) {}
|
||||
|
||||
public:
|
||||
size_t getNumOperands() const { return NumOperands; }
|
||||
|
||||
const SCEV *getOperand(unsigned i) const {
|
||||
SCEVUse getOperand(unsigned i) const {
|
||||
assert(i < NumOperands && "Operand index out of range!");
|
||||
return Operands[i];
|
||||
}
|
||||
|
||||
ArrayRef<const SCEV *> operands() const {
|
||||
return ArrayRef(Operands, NumOperands);
|
||||
}
|
||||
ArrayRef<SCEVUse> operands() const { return ArrayRef(Operands, NumOperands); }
|
||||
|
||||
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask = NoWrapMask) const {
|
||||
return (NoWrapFlags)(SubclassData & Mask);
|
||||
@ -256,13 +255,14 @@ public:
|
||||
S->getSCEVType() == scSequentialUMinExpr ||
|
||||
S->getSCEVType() == scAddRecExpr;
|
||||
}
|
||||
static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
|
||||
};
|
||||
|
||||
/// This node is the base class for n'ary commutative operators.
|
||||
class SCEVCommutativeExpr : public SCEVNAryExpr {
|
||||
protected:
|
||||
SCEVCommutativeExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
|
||||
const SCEV *const *O, size_t N)
|
||||
const SCEVUse *O, size_t N)
|
||||
: SCEVNAryExpr(ID, T, O, N) {}
|
||||
|
||||
public:
|
||||
@ -283,11 +283,10 @@ class SCEVAddExpr : public SCEVCommutativeExpr {
|
||||
|
||||
Type *Ty;
|
||||
|
||||
SCEVAddExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
|
||||
SCEVAddExpr(const FoldingSetNodeIDRef ID, const SCEVUse *O, size_t N)
|
||||
: SCEVCommutativeExpr(ID, scAddExpr, O, N) {
|
||||
auto *FirstPointerTypedOp = find_if(operands(), [](const SCEV *Op) {
|
||||
return Op->getType()->isPointerTy();
|
||||
});
|
||||
auto *FirstPointerTypedOp = find_if(
|
||||
operands(), [](SCEVUse Op) { return Op->getType()->isPointerTy(); });
|
||||
if (FirstPointerTypedOp != operands().end())
|
||||
Ty = (*FirstPointerTypedOp)->getType();
|
||||
else
|
||||
@ -299,13 +298,14 @@ public:
|
||||
|
||||
/// Methods for support type inquiry through isa, cast, and dyn_cast:
|
||||
static bool classof(const SCEV *S) { return S->getSCEVType() == scAddExpr; }
|
||||
static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
|
||||
};
|
||||
|
||||
/// This node represents multiplication of some number of SCEVs.
|
||||
class SCEVMulExpr : public SCEVCommutativeExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVMulExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
|
||||
SCEVMulExpr(const FoldingSetNodeIDRef ID, const SCEVUse *O, size_t N)
|
||||
: SCEVCommutativeExpr(ID, scMulExpr, O, N) {}
|
||||
|
||||
public:
|
||||
@ -313,30 +313,31 @@ public:
|
||||
|
||||
/// Methods for support type inquiry through isa, cast, and dyn_cast:
|
||||
static bool classof(const SCEV *S) { return S->getSCEVType() == scMulExpr; }
|
||||
static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
|
||||
};
|
||||
|
||||
/// This class represents a binary unsigned division operation.
|
||||
class SCEVUDivExpr : public SCEV {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
std::array<const SCEV *, 2> Operands;
|
||||
std::array<SCEVUse, 2> Operands;
|
||||
|
||||
SCEVUDivExpr(const FoldingSetNodeIDRef ID, const SCEV *lhs, const SCEV *rhs)
|
||||
SCEVUDivExpr(const FoldingSetNodeIDRef ID, SCEVUse lhs, SCEVUse rhs)
|
||||
: SCEV(ID, scUDivExpr, computeExpressionSize({lhs, rhs})) {
|
||||
Operands[0] = lhs;
|
||||
Operands[1] = rhs;
|
||||
}
|
||||
|
||||
public:
|
||||
const SCEV *getLHS() const { return Operands[0]; }
|
||||
const SCEV *getRHS() const { return Operands[1]; }
|
||||
SCEVUse getLHS() const { return Operands[0]; }
|
||||
SCEVUse getRHS() const { return Operands[1]; }
|
||||
size_t getNumOperands() const { return 2; }
|
||||
const SCEV *getOperand(unsigned i) const {
|
||||
SCEVUse getOperand(unsigned i) const {
|
||||
assert((i == 0 || i == 1) && "Operand index out of range!");
|
||||
return i == 0 ? getLHS() : getRHS();
|
||||
}
|
||||
|
||||
ArrayRef<const SCEV *> operands() const { return Operands; }
|
||||
ArrayRef<SCEVUse> operands() const { return Operands; }
|
||||
|
||||
Type *getType() const {
|
||||
// In most cases the types of LHS and RHS will be the same, but in some
|
||||
@ -364,25 +365,24 @@ class SCEVAddRecExpr : public SCEVNAryExpr {
|
||||
|
||||
const Loop *L;
|
||||
|
||||
SCEVAddRecExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N,
|
||||
SCEVAddRecExpr(const FoldingSetNodeIDRef ID, const SCEVUse *O, size_t N,
|
||||
const Loop *l)
|
||||
: SCEVNAryExpr(ID, scAddRecExpr, O, N), L(l) {}
|
||||
|
||||
public:
|
||||
Type *getType() const { return getStart()->getType(); }
|
||||
const SCEV *getStart() const { return Operands[0]; }
|
||||
SCEVUse getStart() const { return Operands[0]; }
|
||||
const Loop *getLoop() const { return L; }
|
||||
|
||||
/// Constructs and returns the recurrence indicating how much this
|
||||
/// expression steps by. If this is a polynomial of degree N, it
|
||||
/// returns a chrec of degree N-1. We cannot determine whether
|
||||
/// the step recurrence has self-wraparound.
|
||||
const SCEV *getStepRecurrence(ScalarEvolution &SE) const {
|
||||
SCEVUse getStepRecurrence(ScalarEvolution &SE) const {
|
||||
if (isAffine())
|
||||
return getOperand(1);
|
||||
return SE.getAddRecExpr(
|
||||
SmallVector<const SCEV *, 3>(operands().drop_front()), getLoop(),
|
||||
FlagAnyWrap);
|
||||
return SE.getAddRecExpr(SmallVector<SCEVUse, 3>(operands().drop_front()),
|
||||
getLoop(), FlagAnyWrap);
|
||||
}
|
||||
|
||||
/// Return true if this represents an expression A + B*x where A
|
||||
@ -414,9 +414,9 @@ public:
|
||||
|
||||
/// Return the value of this chain of recurrences at the specified iteration
|
||||
/// number. Takes an explicit list of operands to represent an AddRec.
|
||||
LLVM_ABI static const SCEV *
|
||||
evaluateAtIteration(ArrayRef<const SCEV *> Operands, const SCEV *It,
|
||||
ScalarEvolution &SE);
|
||||
LLVM_ABI static const SCEV *evaluateAtIteration(ArrayRef<SCEVUse> Operands,
|
||||
const SCEV *It,
|
||||
ScalarEvolution &SE);
|
||||
|
||||
/// Return the number of iterations of this loop that produce
|
||||
/// values in the specified constant range. Another way of
|
||||
@ -449,7 +449,7 @@ class SCEVMinMaxExpr : public SCEVCommutativeExpr {
|
||||
protected:
|
||||
/// Note: Constructing subclasses via this constructor is allowed
|
||||
SCEVMinMaxExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
|
||||
const SCEV *const *O, size_t N)
|
||||
const SCEVUse *O, size_t N)
|
||||
: SCEVCommutativeExpr(ID, T, O, N) {
|
||||
assert(isMinMaxType(T));
|
||||
// Min and max never overflow
|
||||
@ -481,7 +481,7 @@ public:
|
||||
class SCEVSMaxExpr : public SCEVMinMaxExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVSMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
|
||||
SCEVSMaxExpr(const FoldingSetNodeIDRef ID, const SCEVUse *O, size_t N)
|
||||
: SCEVMinMaxExpr(ID, scSMaxExpr, O, N) {}
|
||||
|
||||
public:
|
||||
@ -493,7 +493,7 @@ public:
|
||||
class SCEVUMaxExpr : public SCEVMinMaxExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVUMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
|
||||
SCEVUMaxExpr(const FoldingSetNodeIDRef ID, const SCEVUse *O, size_t N)
|
||||
: SCEVMinMaxExpr(ID, scUMaxExpr, O, N) {}
|
||||
|
||||
public:
|
||||
@ -505,7 +505,7 @@ public:
|
||||
class SCEVSMinExpr : public SCEVMinMaxExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVSMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
|
||||
SCEVSMinExpr(const FoldingSetNodeIDRef ID, const SCEVUse *O, size_t N)
|
||||
: SCEVMinMaxExpr(ID, scSMinExpr, O, N) {}
|
||||
|
||||
public:
|
||||
@ -517,7 +517,7 @@ public:
|
||||
class SCEVUMinExpr : public SCEVMinMaxExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVUMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N)
|
||||
SCEVUMinExpr(const FoldingSetNodeIDRef ID, const SCEVUse *O, size_t N)
|
||||
: SCEVMinMaxExpr(ID, scUMinExpr, O, N) {}
|
||||
|
||||
public:
|
||||
@ -544,7 +544,7 @@ class SCEVSequentialMinMaxExpr : public SCEVNAryExpr {
|
||||
protected:
|
||||
/// Note: Constructing subclasses via this constructor is allowed
|
||||
SCEVSequentialMinMaxExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
|
||||
const SCEV *const *O, size_t N)
|
||||
const SCEVUse *O, size_t N)
|
||||
: SCEVNAryExpr(ID, T, O, N) {
|
||||
assert(isSequentialMinMaxType(T));
|
||||
// Min and max never overflow
|
||||
@ -571,13 +571,14 @@ public:
|
||||
static bool classof(const SCEV *S) {
|
||||
return isSequentialMinMaxType(S->getSCEVType());
|
||||
}
|
||||
static bool classof(const SCEVUse *U) { return classof(U->getPointer()); }
|
||||
};
|
||||
|
||||
/// This class represents a sequential/in-order unsigned minimum selection.
|
||||
class SCEVSequentialUMinExpr : public SCEVSequentialMinMaxExpr {
|
||||
friend class ScalarEvolution;
|
||||
|
||||
SCEVSequentialUMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O,
|
||||
SCEVSequentialUMinExpr(const FoldingSetNodeIDRef ID, const SCEVUse *O,
|
||||
size_t N)
|
||||
: SCEVSequentialMinMaxExpr(ID, scSequentialUMinExpr, O, N) {}
|
||||
|
||||
@ -715,7 +716,7 @@ public:
|
||||
case scUMinExpr:
|
||||
case scSequentialUMinExpr:
|
||||
case scAddRecExpr:
|
||||
for (const auto *Op : S->operands()) {
|
||||
for (const SCEV *Op : S->operands()) {
|
||||
push(Op);
|
||||
if (Visitor.isDone())
|
||||
break;
|
||||
@ -825,9 +826,9 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
bool Changed = false;
|
||||
for (const auto *Op : Expr->operands()) {
|
||||
for (const SCEV *Op : Expr->operands()) {
|
||||
Operands.push_back(((SC *)this)->visit(Op));
|
||||
Changed |= Op != Operands.back();
|
||||
}
|
||||
@ -835,9 +836,9 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
bool Changed = false;
|
||||
for (const auto *Op : Expr->operands()) {
|
||||
for (const SCEV *Op : Expr->operands()) {
|
||||
Operands.push_back(((SC *)this)->visit(Op));
|
||||
Changed |= Op != Operands.back();
|
||||
}
|
||||
@ -852,9 +853,9 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
bool Changed = false;
|
||||
for (const auto *Op : Expr->operands()) {
|
||||
for (const SCEV *Op : Expr->operands()) {
|
||||
Operands.push_back(((SC *)this)->visit(Op));
|
||||
Changed |= Op != Operands.back();
|
||||
}
|
||||
@ -864,9 +865,9 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitSMaxExpr(const SCEVSMaxExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
bool Changed = false;
|
||||
for (const auto *Op : Expr->operands()) {
|
||||
for (const SCEV *Op : Expr->operands()) {
|
||||
Operands.push_back(((SC *)this)->visit(Op));
|
||||
Changed |= Op != Operands.back();
|
||||
}
|
||||
@ -874,9 +875,9 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitUMaxExpr(const SCEVUMaxExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
bool Changed = false;
|
||||
for (const auto *Op : Expr->operands()) {
|
||||
for (const SCEV *Op : Expr->operands()) {
|
||||
Operands.push_back(((SC *)this)->visit(Op));
|
||||
Changed |= Op != Operands.back();
|
||||
}
|
||||
@ -884,9 +885,9 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
bool Changed = false;
|
||||
for (const auto *Op : Expr->operands()) {
|
||||
for (const SCEV *Op : Expr->operands()) {
|
||||
Operands.push_back(((SC *)this)->visit(Op));
|
||||
Changed |= Op != Operands.back();
|
||||
}
|
||||
@ -894,9 +895,9 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
bool Changed = false;
|
||||
for (const auto *Op : Expr->operands()) {
|
||||
for (const SCEV *Op : Expr->operands()) {
|
||||
Operands.push_back(((SC *)this)->visit(Op));
|
||||
Changed |= Op != Operands.back();
|
||||
}
|
||||
@ -904,9 +905,9 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
bool Changed = false;
|
||||
for (const auto *Op : Expr->operands()) {
|
||||
for (const SCEV *Op : Expr->operands()) {
|
||||
Operands.push_back(((SC *)this)->visit(Op));
|
||||
Changed |= Op != Operands.back();
|
||||
}
|
||||
@ -964,8 +965,8 @@ public:
|
||||
}
|
||||
|
||||
const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
for (const SCEV *Op : Expr->operands())
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
for (SCEVUse Op : Expr->operands())
|
||||
Operands.push_back(visit(Op));
|
||||
|
||||
const Loop *L = Expr->getLoop();
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
#ifndef LLVM_ANALYSIS_SCALAREVOLUTIONPATTERNMATCH_H
|
||||
#define LLVM_ANALYSIS_SCALAREVOLUTIONPATTERNMATCH_H
|
||||
|
||||
#include "llvm/Analysis/ScalarEvolution.h"
|
||||
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
|
||||
|
||||
namespace llvm {
|
||||
@ -22,6 +23,11 @@ template <typename Pattern> bool match(const SCEV *S, const Pattern &P) {
|
||||
return P.match(S);
|
||||
}
|
||||
|
||||
template <typename Pattern> bool match(const SCEVUse U, const Pattern &P) {
|
||||
const SCEV *S = U.getPointer();
|
||||
return const_cast<Pattern &>(P).match(S);
|
||||
}
|
||||
|
||||
template <typename Predicate> struct cst_pred_ty : public Predicate {
|
||||
cst_pred_ty() = default;
|
||||
cst_pred_ty(uint64_t V) : Predicate(V) {}
|
||||
@ -159,7 +165,8 @@ template <typename SCEVTy, typename Op0_t> struct SCEVUnaryExpr_match {
|
||||
|
||||
bool match(const SCEV *S) const {
|
||||
auto *E = dyn_cast<SCEVTy>(S);
|
||||
return E && E->getNumOperands() == 1 && Op0.match(E->getOperand(0));
|
||||
return E && E->getNumOperands() == 1 &&
|
||||
Op0.match(E->getOperand(0).getPointer());
|
||||
}
|
||||
};
|
||||
|
||||
@ -215,9 +222,10 @@ struct SCEVBinaryExpr_match {
|
||||
|
||||
auto *E = dyn_cast<SCEVTy>(S);
|
||||
return E && E->getNumOperands() == 2 &&
|
||||
((Op0.match(E->getOperand(0)) && Op1.match(E->getOperand(1))) ||
|
||||
(Commutable && Op0.match(E->getOperand(1)) &&
|
||||
Op1.match(E->getOperand(0))));
|
||||
((Op0.match(E->getOperand(0).getPointer()) &&
|
||||
Op1.match(E->getOperand(1).getPointer())) ||
|
||||
(Commutable && Op0.match(E->getOperand(1).getPointer()) &&
|
||||
Op1.match(E->getOperand(0).getPointer())));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -80,6 +80,7 @@
|
||||
|
||||
#include "llvm/ADT/DenseMap.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/Analysis/ScalarEvolution.h"
|
||||
#include "llvm/IR/PassManager.h"
|
||||
#include "llvm/IR/ValueHandle.h"
|
||||
|
||||
@ -92,8 +93,6 @@ class DominatorTree;
|
||||
class Function;
|
||||
class GetElementPtrInst;
|
||||
class Instruction;
|
||||
class ScalarEvolution;
|
||||
class SCEV;
|
||||
class TargetLibraryInfo;
|
||||
class TargetTransformInfo;
|
||||
class Type;
|
||||
@ -114,7 +113,7 @@ private:
|
||||
bool doOneIteration(Function &F);
|
||||
|
||||
// Reassociates I for better CSE.
|
||||
Instruction *tryReassociate(Instruction *I, const SCEV *&OrigSCEV);
|
||||
Instruction *tryReassociate(Instruction *I, SCEVUse &OrigSCEV);
|
||||
|
||||
// Reassociate GEP for better CSE.
|
||||
Instruction *tryReassociateGEP(GetElementPtrInst *GEP);
|
||||
@ -143,19 +142,18 @@ private:
|
||||
Instruction *tryReassociateBinaryOp(Value *LHS, Value *RHS,
|
||||
BinaryOperator *I);
|
||||
// Rewrites I to (LHS op RHS) if LHS is computed already.
|
||||
Instruction *tryReassociatedBinaryOp(const SCEV *LHS, Value *RHS,
|
||||
Instruction *tryReassociatedBinaryOp(SCEVUse LHS, Value *RHS,
|
||||
BinaryOperator *I);
|
||||
|
||||
// Tries to match Op1 and Op2 by using V.
|
||||
bool matchTernaryOp(BinaryOperator *I, Value *V, Value *&Op1, Value *&Op2);
|
||||
|
||||
// Gets SCEV for (LHS op RHS).
|
||||
const SCEV *getBinarySCEV(BinaryOperator *I, const SCEV *LHS,
|
||||
const SCEV *RHS);
|
||||
SCEVUse getBinarySCEV(BinaryOperator *I, SCEVUse LHS, SCEVUse RHS);
|
||||
|
||||
// Returns the closest dominator of \c Dominatee that computes
|
||||
// \c CandidateExpr. Returns null if not found.
|
||||
Instruction *findClosestMatchingDominator(const SCEV *CandidateExpr,
|
||||
Instruction *findClosestMatchingDominator(SCEVUse CandidateExpr,
|
||||
Instruction *Dominatee);
|
||||
|
||||
// Try to match \p I as signed/unsigned Min/Max and reassociate it. \p
|
||||
@ -163,8 +161,7 @@ private:
|
||||
// done or not. If reassociation was successful newly generated instruction is
|
||||
// returned, otherwise nullptr.
|
||||
template <typename PredT>
|
||||
Instruction *matchAndReassociateMinOrMax(Instruction *I,
|
||||
const SCEV *&OrigSCEV);
|
||||
Instruction *matchAndReassociateMinOrMax(Instruction *I, SCEVUse &OrigSCEV);
|
||||
|
||||
// Reassociate Min/Max.
|
||||
template <typename MaxMinT>
|
||||
|
||||
@ -133,7 +133,7 @@ struct SCEVCollectAddRecMultiplies {
|
||||
bool follow(const SCEV *S) {
|
||||
if (auto *Mul = dyn_cast<SCEVMulExpr>(S)) {
|
||||
bool HasAddRec = false;
|
||||
SmallVector<const SCEV *, 0> Operands;
|
||||
SmallVector<SCEVUse, 0> Operands;
|
||||
for (const SCEV *Op : Mul->operands()) {
|
||||
const SCEVUnknown *Unknown = dyn_cast<SCEVUnknown>(Op);
|
||||
if (Unknown && !isa<CallInst>(Unknown->getValue())) {
|
||||
@ -207,7 +207,7 @@ static bool findArrayDimensionsRec(ScalarEvolution &SE,
|
||||
// End of recursion.
|
||||
if (Last == 0) {
|
||||
if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Step)) {
|
||||
SmallVector<const SCEV *, 2> Qs;
|
||||
SmallVector<SCEVUse, 2> Qs;
|
||||
for (const SCEV *Op : M->operands())
|
||||
if (!isa<SCEVConstant>(Op))
|
||||
Qs.push_back(Op);
|
||||
@ -266,7 +266,7 @@ static const SCEV *removeConstantFactors(ScalarEvolution &SE, const SCEV *T) {
|
||||
return T;
|
||||
|
||||
if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(T)) {
|
||||
SmallVector<const SCEV *, 2> Factors;
|
||||
SmallVector<SCEVUse, 2> Factors;
|
||||
for (const SCEV *Op : M->operands())
|
||||
if (!isa<SCEVConstant>(Op))
|
||||
Factors.push_back(Op);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -149,7 +149,7 @@ void SCEVDivision::visitAddRecExpr(const SCEVAddRecExpr *Numerator) {
|
||||
}
|
||||
|
||||
void SCEVDivision::visitAddExpr(const SCEVAddExpr *Numerator) {
|
||||
SmallVector<const SCEV *, 2> Qs, Rs;
|
||||
SmallVector<SCEVUse, 2> Qs, Rs;
|
||||
Type *Ty = Denominator->getType();
|
||||
|
||||
for (const SCEV *Op : Numerator->operands()) {
|
||||
@ -175,7 +175,7 @@ void SCEVDivision::visitAddExpr(const SCEVAddExpr *Numerator) {
|
||||
}
|
||||
|
||||
void SCEVDivision::visitMulExpr(const SCEVMulExpr *Numerator) {
|
||||
SmallVector<const SCEV *, 2> Qs;
|
||||
SmallVector<SCEVUse, 2> Qs;
|
||||
Type *Ty = Denominator->getType();
|
||||
|
||||
bool FoundDenominatorTerm = false;
|
||||
|
||||
@ -46,10 +46,10 @@ struct NormalizeDenormalizeRewriter
|
||||
|
||||
const SCEV *
|
||||
NormalizeDenormalizeRewriter::visitAddRecExpr(const SCEVAddRecExpr *AR) {
|
||||
SmallVector<const SCEV *, 8> Operands;
|
||||
SmallVector<SCEVUse, 8> Operands;
|
||||
|
||||
transform(AR->operands(), std::back_inserter(Operands),
|
||||
[&](const SCEV *Op) { return visit(Op); });
|
||||
[&](SCEVUse Op) { return visit(Op.getPointer()); });
|
||||
|
||||
if (!Pred(AR))
|
||||
return SE.getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagAnyWrap);
|
||||
|
||||
@ -2785,7 +2785,7 @@ void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
|
||||
bool Runtime = true;
|
||||
if (ST->hasLOB()) {
|
||||
if (SE.hasLoopInvariantBackedgeTakenCount(L)) {
|
||||
const auto *BETC = SE.getBackedgeTakenCount(L);
|
||||
const SCEV *BETC = SE.getBackedgeTakenCount(L);
|
||||
auto *Outer = L->getOutermostLoop();
|
||||
if ((L != Outer && Outer != L->getParentLoop()) ||
|
||||
(L != Outer && BETC && !SE.isLoopInvariant(BETC, Outer))) {
|
||||
|
||||
@ -1395,9 +1395,9 @@ createReplacement(ICmpInst *ICmp, const Loop *L, BasicBlock *ExitingBB,
|
||||
// wrap". getLoopInvariantExitCondDuringFirstIterations knows how to deal
|
||||
// with umin in a smart way, but umin(a, b) - 1 will likely not simplify.
|
||||
// So we manually construct umin(a - 1, b - 1).
|
||||
SmallVector<const SCEV *, 4> Elements;
|
||||
SmallVector<SCEVUse, 4> Elements;
|
||||
if (auto *UMin = dyn_cast<SCEVUMinExpr>(MaxIter)) {
|
||||
for (const SCEV *Op : UMin->operands())
|
||||
for (SCEVUse Op : UMin->operands())
|
||||
Elements.push_back(SE->getMinusSCEV(Op, SE->getOne(Op->getType())));
|
||||
MaxIter = SE->getUMinFromMismatchedTypes(Elements);
|
||||
} else
|
||||
|
||||
@ -423,7 +423,7 @@ bool InductiveRangeCheck::reassociateSubLHS(
|
||||
auto getExprScaledIfOverflow = [&](Instruction::BinaryOps BinOp,
|
||||
const SCEV *LHS,
|
||||
const SCEV *RHS) -> const SCEV * {
|
||||
const SCEV *(ScalarEvolution::*Operation)(const SCEV *, const SCEV *,
|
||||
const SCEV *(ScalarEvolution::*Operation)(SCEVUse, SCEVUse,
|
||||
SCEV::NoWrapFlags, unsigned);
|
||||
switch (BinOp) {
|
||||
default:
|
||||
|
||||
@ -391,9 +391,10 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
|
||||
|
||||
BasicBlock *BB = P.InsertPt->getParent();
|
||||
SCEVExpander SCEVE(*SE, "prefaddr");
|
||||
const SCEV *NextLSCEV = SE->getAddExpr(P.LSCEVAddRec, SE->getMulExpr(
|
||||
SE->getConstant(P.LSCEVAddRec->getType(), ItersAhead),
|
||||
P.LSCEVAddRec->getStepRecurrence(*SE)));
|
||||
const SCEV *NextLSCEV = SE->getAddExpr(
|
||||
P.LSCEVAddRec,
|
||||
SE->getMulExpr(SE->getConstant(P.LSCEVAddRec->getType(), ItersAhead),
|
||||
P.LSCEVAddRec->getStepRecurrence(*SE)));
|
||||
if (!SCEVE.isSafeToExpand(NextLSCEV))
|
||||
continue;
|
||||
|
||||
|
||||
@ -1132,7 +1132,7 @@ private:
|
||||
|
||||
const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
|
||||
const Loop *ExprL = Expr->getLoop();
|
||||
SmallVector<const SCEV *, 2> Operands;
|
||||
SmallVector<SCEVUse, 2> Operands;
|
||||
if (ExprL == &OldL) {
|
||||
append_range(Operands, Expr->operands());
|
||||
return SE.getAddRecExpr(Operands, &NewL, Expr->getNoWrapFlags());
|
||||
@ -1147,7 +1147,7 @@ private:
|
||||
return visit(Expr->getStart());
|
||||
}
|
||||
|
||||
for (const SCEV *Op : Expr->operands())
|
||||
for (SCEVUse Op : Expr->operands())
|
||||
Operands.push_back(visit(Op));
|
||||
return SE.getAddRecExpr(Operands, ExprL, Expr->getNoWrapFlags());
|
||||
}
|
||||
|
||||
@ -889,7 +889,7 @@ bool LoopIdiomRecognize::processLoopMemSet(MemSetInst *MSI,
|
||||
return false;
|
||||
}
|
||||
|
||||
const SCEV *MemsetSizeSCEV = SE->getSCEV(MSI->getLength());
|
||||
SCEVUse MemsetSizeSCEV = SE->getSCEV(MSI->getLength());
|
||||
|
||||
bool IsNegStride = false;
|
||||
const bool IsConstantSize = isa<ConstantInt>(MSI->getLength());
|
||||
@ -928,9 +928,9 @@ bool LoopIdiomRecognize::processLoopMemSet(MemSetInst *MSI,
|
||||
|
||||
// Compare positive direction PointerStrideSCEV with MemsetSizeSCEV
|
||||
IsNegStride = PointerStrideSCEV->isNonConstantNegative();
|
||||
const SCEV *PositiveStrideSCEV =
|
||||
IsNegStride ? SE->getNegativeSCEV(PointerStrideSCEV)
|
||||
: PointerStrideSCEV;
|
||||
SCEVUse PositiveStrideSCEV =
|
||||
IsNegStride ? SCEVUse(SE->getNegativeSCEV(PointerStrideSCEV))
|
||||
: SCEVUse(PointerStrideSCEV);
|
||||
LLVM_DEBUG(dbgs() << " MemsetSizeSCEV: " << *MemsetSizeSCEV << "\n"
|
||||
<< " PositiveStrideSCEV: " << *PositiveStrideSCEV
|
||||
<< "\n");
|
||||
|
||||
@ -1006,9 +1006,9 @@ static const SCEV *getMinAnalyzeableBackedgeTakenCount(ScalarEvolution &SE,
|
||||
SmallVector<BasicBlock *, 16> ExitingBlocks;
|
||||
L->getExitingBlocks(ExitingBlocks);
|
||||
|
||||
SmallVector<const SCEV *, 4> ExitCounts;
|
||||
SmallVector<SCEVUse, 4> ExitCounts;
|
||||
for (BasicBlock *ExitingBB : ExitingBlocks) {
|
||||
const SCEV *ExitCount = SE.getExitCount(L, ExitingBB);
|
||||
SCEVUse ExitCount = SE.getExitCount(L, ExitingBB);
|
||||
if (isa<SCEVCouldNotCompute>(ExitCount))
|
||||
continue;
|
||||
assert(DT.dominates(ExitingBB, L->getLoopLatch()) &&
|
||||
|
||||
@ -541,9 +541,8 @@ struct Formula {
|
||||
|
||||
/// Recursion helper for initialMatch.
|
||||
static void DoInitialMatch(const SCEV *S, Loop *L,
|
||||
SmallVectorImpl<const SCEV *> &Good,
|
||||
SmallVectorImpl<const SCEV *> &Bad,
|
||||
ScalarEvolution &SE) {
|
||||
SmallVectorImpl<SCEVUse> &Good,
|
||||
SmallVectorImpl<SCEVUse> &Bad, ScalarEvolution &SE) {
|
||||
// Collect expressions which properly dominate the loop header.
|
||||
if (SE.properlyDominates(S, L->getHeader())) {
|
||||
Good.push_back(S);
|
||||
@ -574,11 +573,11 @@ static void DoInitialMatch(const SCEV *S, Loop *L,
|
||||
// Handle a multiplication by -1 (negation) if it didn't fold.
|
||||
if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S))
|
||||
if (Mul->getOperand(0)->isAllOnesValue()) {
|
||||
SmallVector<const SCEV *, 4> Ops(drop_begin(Mul->operands()));
|
||||
SmallVector<SCEVUse, 4> Ops(drop_begin(Mul->operands()));
|
||||
const SCEV *NewMul = SE.getMulExpr(Ops);
|
||||
|
||||
SmallVector<const SCEV *, 4> MyGood;
|
||||
SmallVector<const SCEV *, 4> MyBad;
|
||||
SmallVector<SCEVUse, 4> MyGood;
|
||||
SmallVector<SCEVUse, 4> MyBad;
|
||||
DoInitialMatch(NewMul, L, MyGood, MyBad, SE);
|
||||
const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue(
|
||||
SE.getEffectiveSCEVType(NewMul->getType())));
|
||||
@ -597,8 +596,8 @@ static void DoInitialMatch(const SCEV *S, Loop *L,
|
||||
/// Incorporate loop-variant parts of S into this Formula, attempting to keep
|
||||
/// all loop-invariant and loop-computable values in a single base register.
|
||||
void Formula::initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) {
|
||||
SmallVector<const SCEV *, 4> Good;
|
||||
SmallVector<const SCEV *, 4> Bad;
|
||||
SmallVector<SCEVUse, 4> Good;
|
||||
SmallVector<SCEVUse, 4> Bad;
|
||||
DoInitialMatch(S, L, Good, Bad, SE);
|
||||
if (!Good.empty()) {
|
||||
const SCEV *Sum = SE.getAddExpr(Good);
|
||||
@ -877,7 +876,7 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
|
||||
// Distribute the sdiv over add operands, if the add doesn't overflow.
|
||||
if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) {
|
||||
if (IgnoreSignificantBits || isAddSExtable(Add, SE)) {
|
||||
SmallVector<const SCEV *, 8> Ops;
|
||||
SmallVector<SCEVUse, 8> Ops;
|
||||
for (const SCEV *S : Add->operands()) {
|
||||
const SCEV *Op = getExactSDiv(S, RHS, SE, IgnoreSignificantBits);
|
||||
if (!Op) return nullptr;
|
||||
@ -906,7 +905,7 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
|
||||
}
|
||||
}
|
||||
|
||||
SmallVector<const SCEV *, 4> Ops;
|
||||
SmallVector<SCEVUse, 4> Ops;
|
||||
bool Found = false;
|
||||
for (const SCEV *S : Mul->operands()) {
|
||||
if (!Found)
|
||||
@ -928,7 +927,7 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
|
||||
|
||||
/// If S involves the addition of a constant integer value, return that integer
|
||||
/// value, and mutate S to point to a new SCEV with that value excluded.
|
||||
static Immediate ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
|
||||
static Immediate ExtractImmediate(SCEVUse &S, ScalarEvolution &SE) {
|
||||
const APInt *C;
|
||||
if (match(S, m_scev_APInt(C))) {
|
||||
if (C->getSignificantBits() <= 64) {
|
||||
@ -936,13 +935,13 @@ static Immediate ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
|
||||
return Immediate::getFixed(C->getSExtValue());
|
||||
}
|
||||
} else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
|
||||
SmallVector<const SCEV *, 8> NewOps(Add->operands());
|
||||
SmallVector<SCEVUse, 8> NewOps(Add->operands());
|
||||
Immediate Result = ExtractImmediate(NewOps.front(), SE);
|
||||
if (Result.isNonZero())
|
||||
S = SE.getAddExpr(NewOps);
|
||||
return Result;
|
||||
} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
|
||||
SmallVector<const SCEV *, 8> NewOps(AR->operands());
|
||||
SmallVector<SCEVUse, 8> NewOps(AR->operands());
|
||||
Immediate Result = ExtractImmediate(NewOps.front(), SE);
|
||||
if (Result.isNonZero())
|
||||
S = SE.getAddRecExpr(NewOps, AR->getLoop(),
|
||||
@ -959,20 +958,20 @@ static Immediate ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
|
||||
|
||||
/// If S involves the addition of a GlobalValue address, return that symbol, and
|
||||
/// mutate S to point to a new SCEV with that value excluded.
|
||||
static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) {
|
||||
static GlobalValue *ExtractSymbol(SCEVUse &S, ScalarEvolution &SE) {
|
||||
if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
|
||||
if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) {
|
||||
S = SE.getConstant(GV->getType(), 0);
|
||||
return GV;
|
||||
}
|
||||
} else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
|
||||
SmallVector<const SCEV *, 8> NewOps(Add->operands());
|
||||
SmallVector<SCEVUse, 8> NewOps(Add->operands());
|
||||
GlobalValue *Result = ExtractSymbol(NewOps.back(), SE);
|
||||
if (Result)
|
||||
S = SE.getAddExpr(NewOps);
|
||||
return Result;
|
||||
} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
|
||||
SmallVector<const SCEV *, 8> NewOps(AR->operands());
|
||||
SmallVector<SCEVUse, 8> NewOps(AR->operands());
|
||||
GlobalValue *Result = ExtractSymbol(NewOps.front(), SE);
|
||||
if (Result)
|
||||
S = SE.getAddRecExpr(NewOps, AR->getLoop(),
|
||||
@ -2042,11 +2041,13 @@ static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
|
||||
|
||||
// Conservatively, create an address with an immediate and a
|
||||
// base and a scale.
|
||||
Immediate BaseOffset = ExtractImmediate(S, SE);
|
||||
GlobalValue *BaseGV = ExtractSymbol(S, SE);
|
||||
SCEVUse SCopy = S;
|
||||
Immediate BaseOffset = ExtractImmediate(SCopy, SE);
|
||||
GlobalValue *BaseGV = ExtractSymbol(SCopy, SE);
|
||||
|
||||
// If there's anything else involved, it's not foldable.
|
||||
if (!S->isZero()) return false;
|
||||
if (!SCopy->isZero())
|
||||
return false;
|
||||
|
||||
// Fast-path: zero is always foldable.
|
||||
if (BaseOffset.isZero() && !BaseGV)
|
||||
@ -2816,7 +2817,9 @@ std::pair<size_t, Immediate> LSRInstance::getUse(const SCEV *&Expr,
|
||||
LSRUse::KindType Kind,
|
||||
MemAccessTy AccessTy) {
|
||||
const SCEV *Copy = Expr;
|
||||
Immediate Offset = ExtractImmediate(Expr, SE);
|
||||
SCEVUse ExprUse = Expr;
|
||||
Immediate Offset = ExtractImmediate(ExprUse, SE);
|
||||
Expr = ExprUse;
|
||||
|
||||
// Basic uses can't accept any offset, for example.
|
||||
if (!isAlwaysFoldable(TTI, Kind, AccessTy, /*BaseGV=*/ nullptr,
|
||||
@ -3958,7 +3961,7 @@ void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
|
||||
continue;
|
||||
|
||||
// Collect all operands except *J.
|
||||
SmallVector<const SCEV *, 8> InnerAddOps(std::as_const(AddOps).begin(), J);
|
||||
SmallVector<SCEVUse, 8> InnerAddOps(std::as_const(AddOps).begin(), J);
|
||||
InnerAddOps.append(std::next(J), std::as_const(AddOps).end());
|
||||
|
||||
// Don't leave just a constant behind in a register if the constant could
|
||||
@ -4049,7 +4052,7 @@ void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
|
||||
// Flatten the representation, i.e., reg1 + 1*reg2 => reg1 + reg2, before
|
||||
// processing the formula.
|
||||
Base.unscale();
|
||||
SmallVector<const SCEV *, 4> Ops;
|
||||
SmallVector<SCEVUse, 4> Ops;
|
||||
Formula NewBase = Base;
|
||||
NewBase.BaseRegs.clear();
|
||||
Type *CombinedIntegerType = nullptr;
|
||||
@ -4086,7 +4089,7 @@ void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
|
||||
|
||||
// If we collected at least two registers, generate a formula combining them.
|
||||
if (Ops.size() > 1) {
|
||||
SmallVector<const SCEV *, 4> OpsCopy(Ops); // Don't let SE modify Ops.
|
||||
SmallVector<SCEVUse, 4> OpsCopy(Ops); // Don't let SE modify Ops.
|
||||
GenerateFormula(SE.getAddExpr(OpsCopy));
|
||||
}
|
||||
|
||||
@ -4105,7 +4108,7 @@ void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
|
||||
void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
|
||||
const Formula &Base, size_t Idx,
|
||||
bool IsScaledReg) {
|
||||
const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
|
||||
SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
|
||||
GlobalValue *GV = ExtractSymbol(G, SE);
|
||||
if (G->isZero() || !GV)
|
||||
return;
|
||||
@ -4165,7 +4168,7 @@ void LSRInstance::GenerateConstantOffsetsImpl(
|
||||
}
|
||||
};
|
||||
|
||||
const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
|
||||
SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
|
||||
|
||||
// With constant offsets and constant steps, we can generate pre-inc
|
||||
// accesses by having the offset equal the step. So, for access #0 with a
|
||||
@ -4522,7 +4525,7 @@ void LSRInstance::GenerateCrossUseConstantOffsets() {
|
||||
DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap;
|
||||
SmallVector<const SCEV *, 8> Sequence;
|
||||
for (const SCEV *Use : RegUses) {
|
||||
const SCEV *Reg = Use; // Make a copy for ExtractImmediate to modify.
|
||||
SCEVUse Reg = Use; // Make a copy for ExtractImmediate to modify.
|
||||
Immediate Imm = ExtractImmediate(Reg, SE);
|
||||
auto Pair = Map.try_emplace(Reg);
|
||||
if (Pair.second)
|
||||
@ -5716,7 +5719,7 @@ Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
|
||||
Type *IntTy = SE.getEffectiveSCEVType(Ty);
|
||||
|
||||
// Build up a list of operands to add together to form the full base.
|
||||
SmallVector<const SCEV *, 8> Ops;
|
||||
SmallVector<SCEVUse, 8> Ops;
|
||||
|
||||
// Expand the BaseRegs portion.
|
||||
for (const SCEV *Reg : F.BaseRegs) {
|
||||
|
||||
@ -225,7 +225,7 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
|
||||
for (const auto Node : depth_first(DT)) {
|
||||
BasicBlock *BB = Node->getBlock();
|
||||
for (Instruction &OrigI : *BB) {
|
||||
const SCEV *OrigSCEV = nullptr;
|
||||
SCEVUse OrigSCEV = nullptr;
|
||||
if (Instruction *NewI = tryReassociate(&OrigI, OrigSCEV)) {
|
||||
Changed = true;
|
||||
OrigI.replaceAllUsesWith(NewI);
|
||||
@ -234,7 +234,7 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
|
||||
DeadInsts.push_back(WeakTrackingVH(&OrigI));
|
||||
// Add the rewritten instruction to SeenExprs; the original
|
||||
// instruction is deleted.
|
||||
const SCEV *NewSCEV = SE->getSCEV(NewI);
|
||||
SCEVUse NewSCEV = SE->getSCEV(NewI);
|
||||
SeenExprs[NewSCEV].push_back(WeakTrackingVH(NewI));
|
||||
|
||||
// Ideally, NewSCEV should equal OldSCEV because tryReassociate(I)
|
||||
@ -273,7 +273,7 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
|
||||
template <typename PredT>
|
||||
Instruction *
|
||||
NaryReassociatePass::matchAndReassociateMinOrMax(Instruction *I,
|
||||
const SCEV *&OrigSCEV) {
|
||||
SCEVUse &OrigSCEV) {
|
||||
Value *LHS = nullptr;
|
||||
Value *RHS = nullptr;
|
||||
|
||||
@ -292,8 +292,8 @@ NaryReassociatePass::matchAndReassociateMinOrMax(Instruction *I,
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Instruction *NaryReassociatePass::tryReassociate(Instruction * I,
|
||||
const SCEV *&OrigSCEV) {
|
||||
Instruction *NaryReassociatePass::tryReassociate(Instruction *I,
|
||||
SCEVUse &OrigSCEV) {
|
||||
|
||||
if (!SE->isSCEVable(I->getType()))
|
||||
return nullptr;
|
||||
@ -397,7 +397,7 @@ NaryReassociatePass::tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
|
||||
Value *RHS, Type *IndexedType) {
|
||||
// Look for GEP's closest dominator that has the same SCEV as GEP except that
|
||||
// the I-th index is replaced with LHS.
|
||||
SmallVector<const SCEV *, 4> IndexExprs;
|
||||
SmallVector<SCEVUse, 4> IndexExprs;
|
||||
for (Use &Index : GEP->indices())
|
||||
IndexExprs.push_back(SE->getSCEV(Index));
|
||||
// Replace the I-th index with LHS.
|
||||
@ -414,8 +414,7 @@ NaryReassociatePass::tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
|
||||
// @reassociate_gep_assume for an example of this canonicalization.
|
||||
IndexExprs[I] = SE->getZeroExtendExpr(IndexExprs[I], GEPArgType);
|
||||
}
|
||||
const SCEV *CandidateExpr = SE->getGEPExpr(cast<GEPOperator>(GEP),
|
||||
IndexExprs);
|
||||
SCEVUse CandidateExpr = SE->getGEPExpr(cast<GEPOperator>(GEP), IndexExprs);
|
||||
|
||||
Value *Candidate = findClosestMatchingDominator(CandidateExpr, GEP);
|
||||
if (Candidate == nullptr)
|
||||
@ -481,8 +480,8 @@ Instruction *NaryReassociatePass::tryReassociateBinaryOp(Value *LHS, Value *RHS,
|
||||
if (LHS->hasOneUse() && matchTernaryOp(I, LHS, A, B)) {
|
||||
// I = (A op B) op RHS
|
||||
// = (A op RHS) op B or (B op RHS) op A
|
||||
const SCEV *AExpr = SE->getSCEV(A), *BExpr = SE->getSCEV(B);
|
||||
const SCEV *RHSExpr = SE->getSCEV(RHS);
|
||||
SCEVUse AExpr = SE->getSCEV(A), BExpr = SE->getSCEV(B);
|
||||
SCEVUse RHSExpr = SE->getSCEV(RHS);
|
||||
if (BExpr != RHSExpr) {
|
||||
if (auto *NewI =
|
||||
tryReassociatedBinaryOp(getBinarySCEV(I, AExpr, RHSExpr), B, I))
|
||||
@ -497,7 +496,7 @@ Instruction *NaryReassociatePass::tryReassociateBinaryOp(Value *LHS, Value *RHS,
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Instruction *NaryReassociatePass::tryReassociatedBinaryOp(const SCEV *LHSExpr,
|
||||
Instruction *NaryReassociatePass::tryReassociatedBinaryOp(SCEVUse LHSExpr,
|
||||
Value *RHS,
|
||||
BinaryOperator *I) {
|
||||
// Look for the closest dominator LHS of I that computes LHSExpr, and replace
|
||||
@ -535,9 +534,8 @@ bool NaryReassociatePass::matchTernaryOp(BinaryOperator *I, Value *V,
|
||||
return false;
|
||||
}
|
||||
|
||||
const SCEV *NaryReassociatePass::getBinarySCEV(BinaryOperator *I,
|
||||
const SCEV *LHS,
|
||||
const SCEV *RHS) {
|
||||
SCEVUse NaryReassociatePass::getBinarySCEV(BinaryOperator *I, SCEVUse LHS,
|
||||
SCEVUse RHS) {
|
||||
switch (I->getOpcode()) {
|
||||
case Instruction::Add:
|
||||
return SE->getAddExpr(LHS, RHS);
|
||||
@ -550,7 +548,7 @@ const SCEV *NaryReassociatePass::getBinarySCEV(BinaryOperator *I,
|
||||
}
|
||||
|
||||
Instruction *
|
||||
NaryReassociatePass::findClosestMatchingDominator(const SCEV *CandidateExpr,
|
||||
NaryReassociatePass::findClosestMatchingDominator(SCEVUse CandidateExpr,
|
||||
Instruction *Dominatee) {
|
||||
auto Pos = SeenExprs.find(CandidateExpr);
|
||||
if (Pos == SeenExprs.end())
|
||||
@ -622,12 +620,11 @@ Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
|
||||
}))
|
||||
return nullptr;
|
||||
|
||||
auto tryCombination = [&](Value *A, const SCEV *AExpr, Value *B,
|
||||
const SCEV *BExpr, Value *C,
|
||||
const SCEV *CExpr) -> Value * {
|
||||
SmallVector<const SCEV *, 2> Ops1{BExpr, AExpr};
|
||||
auto tryCombination = [&](Value *A, SCEVUse AExpr, Value *B, SCEVUse BExpr,
|
||||
Value *C, SCEVUse CExpr) -> Value * {
|
||||
SmallVector<SCEVUse, 2> Ops1{BExpr, AExpr};
|
||||
const SCEVTypes SCEVType = convertToSCEVype(m_MaxMin);
|
||||
const SCEV *R1Expr = SE->getMinMaxExpr(SCEVType, Ops1);
|
||||
SCEVUse R1Expr = SE->getMinMaxExpr(SCEVType, Ops1);
|
||||
|
||||
Instruction *R1MinMax = findClosestMatchingDominator(R1Expr, I);
|
||||
|
||||
@ -636,9 +633,8 @@ Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
|
||||
|
||||
LLVM_DEBUG(dbgs() << "NARY: Found common sub-expr: " << *R1MinMax << "\n");
|
||||
|
||||
SmallVector<const SCEV *, 2> Ops2{SE->getUnknown(C),
|
||||
SE->getUnknown(R1MinMax)};
|
||||
const SCEV *R2Expr = SE->getMinMaxExpr(SCEVType, Ops2);
|
||||
SmallVector<SCEVUse, 2> Ops2{SE->getUnknown(C), SE->getUnknown(R1MinMax)};
|
||||
SCEVUse R2Expr = SE->getMinMaxExpr(SCEVType, Ops2);
|
||||
|
||||
SCEVExpander Expander(*SE, "nary-reassociate");
|
||||
Value *NewMinMax = Expander.expandCodeFor(R2Expr, I->getType(), I);
|
||||
@ -649,9 +645,9 @@ Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
|
||||
return NewMinMax;
|
||||
};
|
||||
|
||||
const SCEV *AExpr = SE->getSCEV(A);
|
||||
const SCEV *BExpr = SE->getSCEV(B);
|
||||
const SCEV *RHSExpr = SE->getSCEV(RHS);
|
||||
SCEVUse AExpr = SE->getSCEV(A);
|
||||
SCEVUse BExpr = SE->getSCEV(B);
|
||||
SCEVUse RHSExpr = SE->getSCEV(RHS);
|
||||
|
||||
if (BExpr != RHSExpr) {
|
||||
// Try (A op RHS) op B
|
||||
|
||||
@ -1086,7 +1086,7 @@ void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
|
||||
if (GEP->getType()->isVectorTy())
|
||||
return;
|
||||
|
||||
SmallVector<const SCEV *, 4> IndexExprs;
|
||||
SmallVector<SCEVUse, 4> IndexExprs;
|
||||
for (Use &Idx : GEP->indices())
|
||||
IndexExprs.push_back(SE->getSCEV(Idx));
|
||||
|
||||
@ -1095,12 +1095,12 @@ void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
|
||||
if (GTI.isStruct())
|
||||
continue;
|
||||
|
||||
const SCEV *OrigIndexExpr = IndexExprs[I - 1];
|
||||
IndexExprs[I - 1] = SE->getZero(OrigIndexExpr->getType());
|
||||
SCEVUse OrigIndexExpr = IndexExprs[I - 1];
|
||||
IndexExprs[I - 1] = SE->getZero(OrigIndexExpr.getPointer()->getType());
|
||||
|
||||
// The base of this candidate is GEP's base plus the offsets of all
|
||||
// indices except this current one.
|
||||
const SCEV *BaseExpr = SE->getGEPExpr(cast<GEPOperator>(GEP), IndexExprs);
|
||||
SCEVUse BaseExpr = SE->getGEPExpr(cast<GEPOperator>(GEP), IndexExprs);
|
||||
Value *ArrayIdx = GEP->getOperand(I);
|
||||
uint64_t ElementSize = GTI.getSequentialElementStride(*DL);
|
||||
IntegerType *PtrIdxTy = cast<IntegerType>(DL->getIndexType(GEP->getType()));
|
||||
|
||||
@ -563,7 +563,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
|
||||
if (isa<PointerType>(Sum->getType())) {
|
||||
// The running sum expression is a pointer. Try to form a getelementptr
|
||||
// at this level with that as the base.
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (; I != E && I->first == CurLoop; ++I) {
|
||||
// If the operand is SCEVUnknown and not instructions, peek through
|
||||
// it, to enable more of it to be folded into the GEP.
|
||||
@ -1336,7 +1336,7 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
|
||||
if (CanonicalIV &&
|
||||
SE.getTypeSizeInBits(CanonicalIV->getType()) > SE.getTypeSizeInBits(Ty) &&
|
||||
!S->getType()->isPointerTy()) {
|
||||
SmallVector<const SCEV *, 4> NewOps(S->getNumOperands());
|
||||
SmallVector<SCEVUse, 4> NewOps(S->getNumOperands());
|
||||
for (unsigned i = 0, e = S->getNumOperands(); i != e; ++i)
|
||||
NewOps[i] = SE.getAnyExtendExpr(S->getOperand(i), CanonicalIV->getType());
|
||||
Value *V = expand(SE.getAddRecExpr(NewOps, S->getLoop(),
|
||||
@ -1360,7 +1360,7 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
|
||||
S->getNoWrapFlags(SCEV::FlagNUW));
|
||||
}
|
||||
|
||||
SmallVector<const SCEV *, 4> NewOps(S->operands());
|
||||
SmallVector<SCEVUse, 4> NewOps(S->operands());
|
||||
NewOps[0] = SE.getConstant(Ty, 0);
|
||||
const SCEV *Rest = SE.getAddRecExpr(NewOps, L,
|
||||
S->getNoWrapFlags(SCEV::FlagNW));
|
||||
|
||||
@ -537,7 +537,8 @@ public:
|
||||
const SCEV *NewStep =
|
||||
SE.getMulExpr(Step, SE.getConstant(Ty, StepMultiplier));
|
||||
const SCEV *ScaledOffset = SE.getMulExpr(Step, SE.getConstant(Ty, Offset));
|
||||
const SCEV *NewStart = SE.getAddExpr(Expr->getStart(), ScaledOffset);
|
||||
const SCEV *NewStart =
|
||||
SE.getAddExpr(Expr->getStart(), SCEVUse(ScaledOffset));
|
||||
return SE.getAddRecExpr(NewStart, NewStep, TheLoop, SCEV::FlagAnyWrap);
|
||||
}
|
||||
|
||||
|
||||
@ -149,11 +149,10 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
|
||||
}
|
||||
|
||||
// Helper to create SCEVs for binary and unary operations.
|
||||
auto CreateSCEV =
|
||||
[&](ArrayRef<VPValue *> Ops,
|
||||
function_ref<const SCEV *(ArrayRef<const SCEV *>)> CreateFn)
|
||||
auto CreateSCEV = [&](ArrayRef<VPValue *> Ops,
|
||||
function_ref<const SCEV *(ArrayRef<SCEVUse>)> CreateFn)
|
||||
-> const SCEV * {
|
||||
SmallVector<const SCEV *, 2> SCEVOps;
|
||||
SmallVector<SCEVUse, 2> SCEVOps;
|
||||
for (VPValue *Op : Ops) {
|
||||
const SCEV *S = getSCEVExprForVPValue(Op, PSE, L);
|
||||
if (isa<SCEVCouldNotCompute>(S))
|
||||
@ -165,46 +164,46 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
|
||||
|
||||
VPValue *LHSVal, *RHSVal;
|
||||
if (match(V, m_Add(m_VPValue(LHSVal), m_VPValue(RHSVal))))
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getAddExpr(Ops[0], Ops[1], SCEV::FlagAnyWrap, 0);
|
||||
});
|
||||
if (match(V, m_Sub(m_VPValue(LHSVal), m_VPValue(RHSVal))))
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getMinusSCEV(Ops[0], Ops[1], SCEV::FlagAnyWrap, 0);
|
||||
});
|
||||
if (match(V, m_Not(m_VPValue(LHSVal)))) {
|
||||
// not X = xor X, -1 = -1 - X
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getMinusSCEV(SE.getMinusOne(Ops[0]->getType()), Ops[0]);
|
||||
});
|
||||
}
|
||||
if (match(V, m_Mul(m_VPValue(LHSVal), m_VPValue(RHSVal))))
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getMulExpr(Ops[0], Ops[1], SCEV::FlagAnyWrap, 0);
|
||||
});
|
||||
if (match(V,
|
||||
m_Binary<Instruction::UDiv>(m_VPValue(LHSVal), m_VPValue(RHSVal))))
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getUDivExpr(Ops[0], Ops[1]);
|
||||
});
|
||||
// Handle AND with constant mask: x & (2^n - 1) can be represented as x % 2^n.
|
||||
const APInt *Mask;
|
||||
if (match(V, m_c_BinaryAnd(m_VPValue(LHSVal), m_APInt(Mask))) &&
|
||||
(*Mask + 1).isPowerOf2())
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getURemExpr(Ops[0], SE.getConstant(*Mask + 1));
|
||||
});
|
||||
if (match(V, m_Trunc(m_VPValue(LHSVal)))) {
|
||||
const VPlan *Plan = V->getDefiningRecipe()->getParent()->getPlan();
|
||||
Type *DestTy = VPTypeAnalysis(*Plan).inferScalarType(V);
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getTruncateExpr(Ops[0], DestTy);
|
||||
});
|
||||
}
|
||||
if (match(V, m_ZExt(m_VPValue(LHSVal)))) {
|
||||
const VPlan *Plan = V->getDefiningRecipe()->getParent()->getPlan();
|
||||
Type *DestTy = VPTypeAnalysis(*Plan).inferScalarType(V);
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getZeroExtendExpr(Ops[0], DestTy);
|
||||
});
|
||||
}
|
||||
@ -225,35 +224,35 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
|
||||
SE.getSignExtendExpr(V2, DestTy), SCEV::FlagNSW);
|
||||
}
|
||||
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getSignExtendExpr(Ops[0], DestTy);
|
||||
});
|
||||
}
|
||||
if (match(V,
|
||||
m_Intrinsic<Intrinsic::umax>(m_VPValue(LHSVal), m_VPValue(RHSVal))))
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getUMaxExpr(Ops[0], Ops[1]);
|
||||
});
|
||||
if (match(V,
|
||||
m_Intrinsic<Intrinsic::smax>(m_VPValue(LHSVal), m_VPValue(RHSVal))))
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getSMaxExpr(Ops[0], Ops[1]);
|
||||
});
|
||||
if (match(V,
|
||||
m_Intrinsic<Intrinsic::umin>(m_VPValue(LHSVal), m_VPValue(RHSVal))))
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getUMinExpr(Ops[0], Ops[1]);
|
||||
});
|
||||
if (match(V,
|
||||
m_Intrinsic<Intrinsic::smin>(m_VPValue(LHSVal), m_VPValue(RHSVal))))
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<const SCEV *> Ops) {
|
||||
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getSMinExpr(Ops[0], Ops[1]);
|
||||
});
|
||||
|
||||
ArrayRef<VPValue *> Ops;
|
||||
Type *SourceElementType;
|
||||
if (match(V, m_GetElementPtr(SourceElementType, Ops))) {
|
||||
const SCEV *GEPExpr = CreateSCEV(Ops, [&](ArrayRef<const SCEV *> Ops) {
|
||||
const SCEV *GEPExpr = CreateSCEV(Ops, [&](ArrayRef<SCEVUse> Ops) {
|
||||
return SE.getGEPExpr(Ops.front(), Ops.drop_front(), SourceElementType);
|
||||
});
|
||||
return PSE.getPredicatedSCEV(GEPExpr);
|
||||
|
||||
@ -251,12 +251,12 @@ TEST_F(ScalarEvolutionsTest, CommutativeExprOperandOrder) {
|
||||
EXPECT_EQ(SE.getMulExpr(B, C), SE.getMulExpr(C, B));
|
||||
EXPECT_EQ(SE.getMulExpr(A, C), SE.getMulExpr(C, A));
|
||||
|
||||
SmallVector<const SCEV *, 3> Ops0 = {A, B, C};
|
||||
SmallVector<const SCEV *, 3> Ops1 = {A, C, B};
|
||||
SmallVector<const SCEV *, 3> Ops2 = {B, A, C};
|
||||
SmallVector<const SCEV *, 3> Ops3 = {B, C, A};
|
||||
SmallVector<const SCEV *, 3> Ops4 = {C, B, A};
|
||||
SmallVector<const SCEV *, 3> Ops5 = {C, A, B};
|
||||
SmallVector<SCEVUse, 3> Ops0 = {A, B, C};
|
||||
SmallVector<SCEVUse, 3> Ops1 = {A, C, B};
|
||||
SmallVector<SCEVUse, 3> Ops2 = {B, A, C};
|
||||
SmallVector<SCEVUse, 3> Ops3 = {B, C, A};
|
||||
SmallVector<SCEVUse, 3> Ops4 = {C, B, A};
|
||||
SmallVector<SCEVUse, 3> Ops5 = {C, A, B};
|
||||
|
||||
const SCEV *Mul0 = SE.getMulExpr(Ops0);
|
||||
const SCEV *Mul1 = SE.getMulExpr(Ops1);
|
||||
@ -542,13 +542,14 @@ TEST_F(ScalarEvolutionsTest, SCEVNormalization) {
|
||||
auto *L1 = *std::next(LI.begin());
|
||||
auto *L0 = *std::next(LI.begin(), 2);
|
||||
|
||||
auto GetAddRec = [&SE](const Loop *L, std::initializer_list<const SCEV *> Ops) {
|
||||
SmallVector<const SCEV *, 4> OpsCopy(Ops);
|
||||
auto GetAddRec = [&SE](const Loop *L,
|
||||
std::initializer_list<const SCEV *> Ops) {
|
||||
SmallVector<SCEVUse, 4> OpsCopy(Ops.begin(), Ops.end());
|
||||
return SE.getAddRecExpr(OpsCopy, L, SCEV::FlagAnyWrap);
|
||||
};
|
||||
|
||||
auto GetAdd = [&SE](std::initializer_list<const SCEV *> Ops) {
|
||||
SmallVector<const SCEV *, 4> OpsCopy(Ops);
|
||||
SmallVector<SCEVUse, 4> OpsCopy(Ops.begin(), Ops.end());
|
||||
return SE.getAddExpr(OpsCopy, SCEV::FlagAnyWrap);
|
||||
};
|
||||
|
||||
@ -1729,7 +1730,7 @@ TEST_F(ScalarEvolutionsTest, ComplexityComparatorIsStrictWeakOrdering2) {
|
||||
const SCEV *M0 = SE.getNegativeSCEV(P0);
|
||||
const SCEV *M2 = SE.getNegativeSCEV(P2);
|
||||
|
||||
SmallVector<const SCEV *, 6> Ops = {M2, P0, M0, P1, P2};
|
||||
SmallVector<SCEVUse, 6> Ops = {M2, P0, M0, P1, P2};
|
||||
// When _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG, this will
|
||||
// crash if the comparator has the specific caching bug.
|
||||
SE.getAddExpr(Ops);
|
||||
|
||||
@ -873,8 +873,8 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpandNonAffineAddRec) {
|
||||
|
||||
// Expand {5,+,1,+,1}
|
||||
auto GetAR3 = [&](ScalarEvolution &SE, Loop *L) -> const SCEVAddRecExpr * {
|
||||
SmallVector<const SCEV *, 3> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
|
||||
SE.getOne(ARType), SE.getOne(ARType)};
|
||||
SmallVector<SCEVUse, 3> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
|
||||
SE.getOne(ARType), SE.getOne(ARType)};
|
||||
return cast<SCEVAddRecExpr>(SE.getAddRecExpr(Ops, L, SCEV::FlagAnyWrap));
|
||||
};
|
||||
TestNoCanonicalIV(GetAR3);
|
||||
@ -883,9 +883,9 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpandNonAffineAddRec) {
|
||||
|
||||
// Expand {5,+,1,+,1,+,1}
|
||||
auto GetAR4 = [&](ScalarEvolution &SE, Loop *L) -> const SCEVAddRecExpr * {
|
||||
SmallVector<const SCEV *, 4> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
|
||||
SE.getOne(ARType), SE.getOne(ARType),
|
||||
SE.getOne(ARType)};
|
||||
SmallVector<SCEVUse, 4> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
|
||||
SE.getOne(ARType), SE.getOne(ARType),
|
||||
SE.getOne(ARType)};
|
||||
return cast<SCEVAddRecExpr>(SE.getAddRecExpr(Ops, L, SCEV::FlagAnyWrap));
|
||||
};
|
||||
TestNoCanonicalIV(GetAR4);
|
||||
@ -894,9 +894,9 @@ TEST_F(ScalarEvolutionExpanderTest, SCEVExpandNonAffineAddRec) {
|
||||
|
||||
// Expand {5,+,1,+,1,+,1,+,1}
|
||||
auto GetAR5 = [&](ScalarEvolution &SE, Loop *L) -> const SCEVAddRecExpr * {
|
||||
SmallVector<const SCEV *, 5> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
|
||||
SE.getOne(ARType), SE.getOne(ARType),
|
||||
SE.getOne(ARType), SE.getOne(ARType)};
|
||||
SmallVector<SCEVUse, 5> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
|
||||
SE.getOne(ARType), SE.getOne(ARType),
|
||||
SE.getOne(ARType), SE.getOne(ARType)};
|
||||
return cast<SCEVAddRecExpr>(SE.getAddRecExpr(Ops, L, SCEV::FlagAnyWrap));
|
||||
};
|
||||
TestNoCanonicalIV(GetAR5);
|
||||
|
||||
@ -35,7 +35,6 @@ using llvm::Instruction;
|
||||
using llvm::LoadInst;
|
||||
using llvm::Loop;
|
||||
using llvm::LoopInfo;
|
||||
using llvm::LoopToScevMapT;
|
||||
using llvm::MapVector;
|
||||
using llvm::PHINode;
|
||||
using llvm::ScalarEvolution;
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
#define POLLY_SUPPORT_IRHELPER_H
|
||||
|
||||
#include "llvm/ADT/SetVector.h"
|
||||
#include "llvm/Analysis/ScalarEvolution.h"
|
||||
#include "llvm/IR/Instructions.h"
|
||||
#include "llvm/IR/IntrinsicInst.h"
|
||||
#include "llvm/IR/ValueHandle.h"
|
||||
@ -37,7 +38,7 @@ class Scop;
|
||||
class ScopStmt;
|
||||
|
||||
/// Same as llvm/Analysis/ScalarEvolutionExpressions.h
|
||||
using LoopToScevMapT = llvm::DenseMap<const llvm::Loop *, const llvm::SCEV *>;
|
||||
using LoopToScevMapT = llvm::DenseMap<const llvm::Loop *, llvm::SCEVUse>;
|
||||
|
||||
/// Enumeration of assumptions Polly can take.
|
||||
enum AssumptionKind {
|
||||
|
||||
@ -886,7 +886,7 @@ ScopDetection::getDelinearizationTerms(DetectionContext &Context,
|
||||
if (auto *AF2 = dyn_cast<SCEVAddRecExpr>(Op))
|
||||
collectParametricTerms(SE, AF2, Terms);
|
||||
if (auto *AF2 = dyn_cast<SCEVMulExpr>(Op)) {
|
||||
SmallVector<const SCEV *, 0> Operands;
|
||||
SmallVector<SCEVUse, 0> Operands;
|
||||
|
||||
for (const SCEV *MulOp : AF2->operands()) {
|
||||
if (auto *Const = dyn_cast<SCEVConstant>(MulOp))
|
||||
|
||||
@ -700,7 +700,7 @@ polly::extractConstantFactor(const SCEV *S, ScalarEvolution &SE) {
|
||||
}
|
||||
|
||||
if (auto *Add = dyn_cast<SCEVAddExpr>(S)) {
|
||||
SmallVector<const SCEV *, 4> LeftOvers;
|
||||
SmallVector<SCEVUse, 4> LeftOvers;
|
||||
auto Op0Pair = extractConstantFactor(Add->getOperand(0), SE);
|
||||
auto *Factor = Op0Pair.first;
|
||||
if (SE.isKnownNegative(Factor)) {
|
||||
@ -729,7 +729,7 @@ polly::extractConstantFactor(const SCEV *S, ScalarEvolution &SE) {
|
||||
if (!Mul)
|
||||
return std::make_pair(ConstPart, S);
|
||||
|
||||
SmallVector<const SCEV *, 4> LeftOvers;
|
||||
SmallVector<SCEVUse, 4> LeftOvers;
|
||||
for (const SCEV *Op : Mul->operands())
|
||||
if (isa<SCEVConstant>(Op))
|
||||
ConstPart = cast<SCEVConstant>(SE.getMulExpr(ConstPart, Op));
|
||||
|
||||
@ -243,7 +243,7 @@ struct ScopExpander final : SCEVVisitor<ScopExpander, const SCEV *> {
|
||||
|
||||
explicit ScopExpander(const Region &R, ScalarEvolution &SE, Function *GenFn,
|
||||
ScalarEvolution &GenSE, const char *Name,
|
||||
ValueMapT *VMap, LoopToScevMapT *LoopMap,
|
||||
ValueMapT *VMap, polly::LoopToScevMapT *LoopMap,
|
||||
BasicBlock *RTCBB)
|
||||
: Expander(GenSE, Name, /*PreserveLCSSA=*/false), Name(Name), R(R),
|
||||
VMap(VMap), LoopMap(LoopMap), RTCBB(RTCBB), GenSE(GenSE), GenFn(GenFn) {
|
||||
@ -272,7 +272,7 @@ private:
|
||||
const char *Name;
|
||||
const Region &R;
|
||||
ValueMapT *VMap;
|
||||
LoopToScevMapT *LoopMap;
|
||||
polly::LoopToScevMapT *LoopMap;
|
||||
BasicBlock *RTCBB;
|
||||
DenseMap<const SCEV *, const SCEV *> SCEVCache;
|
||||
|
||||
@ -389,50 +389,50 @@ private:
|
||||
return GenSE.getUDivExpr(visit(E->getLHS()), RHSScev);
|
||||
}
|
||||
const SCEV *visitAddExpr(const SCEVAddExpr *E) {
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (const SCEV *Op : E->operands())
|
||||
NewOps.push_back(visit(Op));
|
||||
return GenSE.getAddExpr(NewOps);
|
||||
}
|
||||
const SCEV *visitMulExpr(const SCEVMulExpr *E) {
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (const SCEV *Op : E->operands())
|
||||
NewOps.push_back(visit(Op));
|
||||
return GenSE.getMulExpr(NewOps);
|
||||
}
|
||||
const SCEV *visitUMaxExpr(const SCEVUMaxExpr *E) {
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
for (const SCEV *Op : E->operands())
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (SCEVUse Op : E->operands())
|
||||
NewOps.push_back(visit(Op));
|
||||
return GenSE.getUMaxExpr(NewOps);
|
||||
}
|
||||
const SCEV *visitSMaxExpr(const SCEVSMaxExpr *E) {
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
for (const SCEV *Op : E->operands())
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (SCEVUse Op : E->operands())
|
||||
NewOps.push_back(visit(Op));
|
||||
return GenSE.getSMaxExpr(NewOps);
|
||||
}
|
||||
const SCEV *visitUMinExpr(const SCEVUMinExpr *E) {
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
for (const SCEV *Op : E->operands())
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (SCEVUse Op : E->operands())
|
||||
NewOps.push_back(visit(Op));
|
||||
return GenSE.getUMinExpr(NewOps);
|
||||
}
|
||||
const SCEV *visitSMinExpr(const SCEVSMinExpr *E) {
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
for (const SCEV *Op : E->operands())
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (SCEVUse Op : E->operands())
|
||||
NewOps.push_back(visit(Op));
|
||||
return GenSE.getSMinExpr(NewOps);
|
||||
}
|
||||
const SCEV *visitSequentialUMinExpr(const SCEVSequentialUMinExpr *E) {
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
for (const SCEV *Op : E->operands())
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (SCEVUse Op : E->operands())
|
||||
NewOps.push_back(visit(Op));
|
||||
return GenSE.getUMinExpr(NewOps, /*Sequential=*/true);
|
||||
}
|
||||
const SCEV *visitAddRecExpr(const SCEVAddRecExpr *E) {
|
||||
SmallVector<const SCEV *, 4> NewOps;
|
||||
for (const SCEV *Op : E->operands())
|
||||
SmallVector<SCEVUse, 4> NewOps;
|
||||
for (SCEVUse Op : E->operands())
|
||||
NewOps.push_back(visit(Op));
|
||||
|
||||
const Loop *L = E->getLoop();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user