shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[SCEV] Move NoWrapFlags definition outside SCEV scope, use for SCEVUse. (#190199)

Browse Source 
The patch moves out of SCEV's scope so they can be re-used for SCEVUse.
SCEVUse gets an additional getNoWrapFlags helper that returns the union
of the expressions SCEV flags and the use-specific flags.

SCEVExpander has been updated to use this new helper.

In order to avoid other changes, the original names are exposed via
constexpr in SCEV. Not sure if there's a nicer way. One alternative
would be to define the enum in struct, and have SCEV inherit from it.

The patch also clarifies that the SCEVUse flags encode NUW/NSW, and
hides getInt, setInt, etc to avoid potential mis-use.

PR: https://github.com/llvm/llvm-project/pull/190199
This commit is contained in:
Florian Hahn 2026-04-04 16:03:36 +01:00 committed by GitHub
parent 47cd798670
commit ff4c6fe24e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 215 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

177
llvm/include/llvm/Analysis/ScalarEvolution.h
View File

@ -22,6 +22,7 @@
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/FoldingSet.h"
@ -66,39 +67,104 @@ enum SCEVTypes : unsigned short;
LLVM_ABI extern bool VerifySCEV;
/// NoWrapFlags are bitfield indices into SCEV's SubclassData.
///
/// Add and Mul expressions may have no-unsigned-wrap <NUW> or
/// no-signed-wrap <NSW> properties, which are derived from the IR
/// operator. NSW is a misnomer that we use to mean no signed overflow or
/// underflow.
///
/// AddRec expressions may have a no-self-wraparound <NW> property if, in
/// the integer domain, abs(step) * max-iteration(loop) <=
/// unsigned-max(bitwidth). This means that the recurrence will never reach
/// its start value if the step is non-zero. Computing the same value on
/// each iteration is not considered wrapping, and recurrences with step = 0
/// are trivially <NW>. <NW> is independent of the sign of step and the
/// value the add recurrence starts with.
///
/// Note that NUW and NSW are also valid properties of a recurrence, and
/// either implies NW. For convenience, NW will be set for a recurrence
/// whenever either NUW or NSW are set.
///
/// We require that the flag on a SCEV apply to the entire scope in which
/// that SCEV is defined. A SCEV's scope is set of locations dominated by
/// a defining location, which is in turn described by the following rules:
/// * A SCEVUnknown is at the point of definition of the Value.
/// * A SCEVConstant is defined at all points.
/// * A SCEVAddRec is defined starting with the header of the associated
/// loop.
/// * All other SCEVs are defined at the earlest point all operands are
/// defined.
///
/// The above rules describe a maximally hoisted form (without regards to
/// potential control dependence). A SCEV is defined anywhere a
/// corresponding instruction could be defined in said maximally hoisted
/// form. Note that SCEVUDivExpr (currently the only expression type which
/// can trap) can be defined per these rules in regions where it would trap
/// at runtime. A SCEV being defined does not require the existence of any
/// instruction within the defined scope.
enum class SCEVNoWrapFlags {
FlagAnyWrap = 0, // No guarantee.
FlagNW = (1 << 0), // No self-wrap.
FlagNUW = (1 << 1), // No unsigned wrap.
FlagNSW = (1 << 2), // No signed wrap.
NoWrapMask = (1 << 3) - 1,
LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/NoWrapMask)
};
class SCEV;
template <typename SCEVPtrT = const SCEV *>
struct SCEVUseT : PointerIntPair<SCEVPtrT, 2> {
struct SCEVUseT : private PointerIntPair<SCEVPtrT, 2> {
using Base = PointerIntPair<SCEVPtrT, 2>;
using Base::getOpaqueValue;
using Base::getPointer;
SCEVUseT() : Base() { Base::setFromOpaqueValue(nullptr); }
SCEVUseT(SCEVPtrT S) : SCEVUseT(S, 0) {}
SCEVUseT(SCEVPtrT S, unsigned Flags) : Base(S, Flags) {}
SCEVUseT() : Base(nullptr, 0) {}
SCEVUseT(SCEVPtrT S) : Base(S, 0) {}
/// Construct with NoWrapFlags; only NUW/NSW are encoded, NW is dropped.
SCEVUseT(SCEVPtrT S, SCEVNoWrapFlags Flags)
: Base(S, static_cast<unsigned>(Flags) >> 1) {}
template <typename OtherPtrT, typename = std::enable_if_t<
std::is_convertible_v<OtherPtrT, SCEVPtrT>>>
SCEVUseT(const SCEVUseT<OtherPtrT> &Other)
: Base(Other.getPointer(), Other.getInt()) {}
: SCEVUseT(Other.getPointer(), Other.getUseNoWrapFlags()) {}
operator SCEVPtrT() const { return Base::getPointer(); }
SCEVPtrT operator->() const { return Base::getPointer(); }
operator SCEVPtrT() const { return getPointer(); }
SCEVPtrT operator->() const { return getPointer(); }
void *getRawPointer() const { return Base::getOpaqueValue(); }
/// Returns true of the SCEVUse is canonical, i.e. no SCEVUse flags set in any
/// Returns true if the SCEVUse is canonical, i.e. no SCEVUse flags set in any
/// operands.
bool isCanonical() const { return getCanonical() == getRawPointer(); }
bool isCanonical() const { return getCanonical() == getOpaqueValue(); }
/// Return the canonical SCEV for this SCEVUse.
const SCEV *getCanonical() const;
unsigned getFlags() const { return Base::getInt(); }
/// Return the no-wrap flags for this SCEVUse, which is the union of the
/// use-specific flags and the underlying SCEV's flags, masked by \p Mask.
SCEVNoWrapFlags
getNoWrapFlags(SCEVNoWrapFlags Mask = SCEVNoWrapFlags::NoWrapMask) const;
bool operator==(const SCEVUseT &RHS) const {
return getRawPointer() == RHS.getRawPointer();
/// Return only the use-specific no-wrap flags (NUW/NSW) without the
/// underlying SCEV's flags.
SCEVNoWrapFlags getUseNoWrapFlags() const {
SCEVNoWrapFlags UseFlags =
static_cast<SCEVNoWrapFlags>(Base::getInt() << 1);
if (any(UseFlags & (SCEVNoWrapFlags::FlagNUW | SCEVNoWrapFlags::FlagNSW)))
UseFlags |= SCEVNoWrapFlags::FlagNW;
return UseFlags;
}
bool operator==(const SCEV *RHS) const { return getRawPointer() == RHS; }
bool operator==(const SCEVUseT &RHS) const {
return getOpaqueValue() == RHS.getOpaqueValue();
}
bool operator!=(const SCEVUseT &RHS) const { return !(*this == RHS); }
bool operator>(const SCEVUseT &RHS) const { return Base::operator>(RHS); }
bool operator==(const SCEV *RHS) const { return getOpaqueValue() == RHS; }
bool operator!=(const SCEV *RHS) const { return getOpaqueValue() != RHS; }
/// Print out the internal representation of this scalar to the specified
/// stream. This should really only be used for debugging purposes.
@ -106,6 +172,10 @@ struct SCEVUseT : PointerIntPair<SCEVPtrT, 2> {
/// This method is used for debugging.
void dump() const;
private:
using Base::setFromOpaqueValue;
friend struct PointerLikeTypeTraits<SCEVUseT>;
};
/// Deduction guide for various SCEV subclass pointers.
@ -139,11 +209,11 @@ template <> struct DenseMapInfo<SCEVUse> {
}
static unsigned getHashValue(SCEVUse U) {
return hash_value(U.getRawPointer());
return hash_value(U.getOpaqueValue());
}
static bool isEqual(const SCEVUse LHS, const SCEVUse RHS) {
return LHS.getRawPointer() == RHS.getRawPointer();
return LHS.getOpaqueValue() == RHS.getOpaqueValue();
}
};
@ -165,7 +235,7 @@ struct CastInfo<SCEVUseT<ToSCEVPtrT>, SCEVUse,
static bool isPossible(const SCEVUse &U) { return isa<To>(U.getPointer()); }
static CastReturnType doCast(const SCEVUse &U) {
return {cast<To>(U.getPointer()), U.getFlags()};
return CastReturnType(cast<To>(U.getPointer()), U.getUseNoWrapFlags());
}
static CastReturnType castFailed() { return CastReturnType(nullptr); }
static CastReturnType doCastIfPossible(const SCEVUse &U) {
@ -206,49 +276,12 @@ protected:
const SCEV *CanonicalSCEV = nullptr;
public:
/// NoWrapFlags are bitfield indices into SubclassData.
///
/// Add and Mul expressions may have no-unsigned-wrap <NUW> or
/// no-signed-wrap <NSW> properties, which are derived from the IR
/// operator. NSW is a misnomer that we use to mean no signed overflow or
/// underflow.
///
/// AddRec expressions may have a no-self-wraparound <NW> property if, in
/// the integer domain, abs(step) * max-iteration(loop) <=
/// unsigned-max(bitwidth). This means that the recurrence will never reach
/// its start value if the step is non-zero. Computing the same value on
/// each iteration is not considered wrapping, and recurrences with step = 0
/// are trivially <NW>. <NW> is independent of the sign of step and the
/// value the add recurrence starts with.
///
/// Note that NUW and NSW are also valid properties of a recurrence, and
/// either implies NW. For convenience, NW will be set for a recurrence
/// whenever either NUW or NSW are set.
///
/// We require that the flag on a SCEV apply to the entire scope in which
/// that SCEV is defined. A SCEV's scope is set of locations dominated by
/// a defining location, which is in turn described by the following rules:
/// * A SCEVUnknown is at the point of definition of the Value.
/// * A SCEVConstant is defined at all points.
/// * A SCEVAddRec is defined starting with the header of the associated
/// loop.
/// * All other SCEVs are defined at the earlest point all operands are
/// defined.
///
/// The above rules describe a maximally hoisted form (without regards to
/// potential control dependence). A SCEV is defined anywhere a
/// corresponding instruction could be defined in said maximally hoisted
/// form. Note that SCEVUDivExpr (currently the only expression type which
/// can trap) can be defined per these rules in regions where it would trap
/// at runtime. A SCEV being defined does not require the existence of any
/// instruction within the defined scope.
enum NoWrapFlags {
FlagAnyWrap = 0, // No guarantee.
FlagNW = (1 << 0), // No self-wrap.
FlagNUW = (1 << 1), // No unsigned wrap.
FlagNSW = (1 << 2), // No signed wrap.
NoWrapMask = (1 << 3) - 1
};
using NoWrapFlags = SCEVNoWrapFlags;
static constexpr auto FlagAnyWrap = SCEVNoWrapFlags::FlagAnyWrap;
static constexpr auto FlagNW = SCEVNoWrapFlags::FlagNW;
static constexpr auto FlagNUW = SCEVNoWrapFlags::FlagNUW;
static constexpr auto FlagNSW = SCEVNoWrapFlags::FlagNSW;
static constexpr auto NoWrapMask = SCEVNoWrapFlags::NoWrapMask;
explicit SCEV(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
unsigned short ExpressionSize)
@ -605,19 +638,19 @@ public:
ProperlyDominatesBlock ///< The SCEV properly dominates the block.
};
/// Convenient NoWrapFlags manipulation that hides enum casts and is
/// visible in the ScalarEvolution name space.
/// Convenient NoWrapFlags manipulation. TODO: Replace with & operator of
/// enum class.
[[nodiscard]] static SCEV::NoWrapFlags maskFlags(SCEV::NoWrapFlags Flags,
int Mask) {
return (SCEV::NoWrapFlags)(Flags & Mask);
SCEV::NoWrapFlags Mask) {
return Flags & Mask;
}
[[nodiscard]] static SCEV::NoWrapFlags setFlags(SCEV::NoWrapFlags Flags,
SCEV::NoWrapFlags OnFlags) {
return (SCEV::NoWrapFlags)(Flags | OnFlags);
return Flags | OnFlags;
}
[[nodiscard]] static SCEV::NoWrapFlags
clearFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags OffFlags) {
return (SCEV::NoWrapFlags)(Flags & ~OffFlags);
return Flags & ~OffFlags;
}
[[nodiscard]] static bool hasFlags(SCEV::NoWrapFlags Flags,
SCEV::NoWrapFlags TestFlags) {
@ -2691,16 +2724,16 @@ template <> struct DenseMapInfo<ScalarEvolution::FoldID> {
};
template <> inline const SCEV *SCEVUseT<const SCEV *>::getCanonical() const {
return Base::getPointer()->getCanonical();
return getPointer()->getCanonical();
}
template <typename SCEVPtrT>
void SCEVUseT<SCEVPtrT>::print(raw_ostream &OS) const {
Base::getPointer()->print(OS);
SCEV::NoWrapFlags Flags = static_cast<SCEV::NoWrapFlags>(Base::getInt());
if (Flags & SCEV::FlagNUW)
getPointer()->print(OS);
SCEV::NoWrapFlags Flags = getUseNoWrapFlags();
if (any(Flags & SCEV::FlagNUW))
OS << "(u nuw)";
if (Flags & SCEV::FlagNSW)
if (any(Flags & SCEV::FlagNSW))
OS << "(u nsw)";
}

27
llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
View File

@ -234,7 +234,7 @@ public:
ArrayRef<SCEVUse> operands() const { return ArrayRef(Operands, NumOperands); }
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask = NoWrapMask) const {
return (NoWrapFlags)(SubclassData & Mask);
return static_cast<NoWrapFlags>(SubclassData) & Mask;
}
bool hasNoUnsignedWrap() const {
@ -274,7 +274,9 @@ public:
}
/// Set flags for a non-recurrence without clearing previously set flags.
void setNoWrapFlags(NoWrapFlags Flags) { SubclassData |= Flags; }
void setNoWrapFlags(NoWrapFlags Flags) {
SubclassData |= static_cast<unsigned short>(Flags);
}
};
/// This node represents an addition of some number of SCEVs.
@ -402,9 +404,9 @@ public:
/// For AddRec, either NUW or NSW implies NW. Keep track of this fact here
/// to make it easier to propagate flags.
void setNoWrapFlags(NoWrapFlags Flags) {
if (Flags & (FlagNUW | FlagNSW))
if (any(Flags & (FlagNUW | FlagNSW)))
Flags = ScalarEvolution::setFlags(Flags, FlagNW);
SubclassData |= Flags;
SubclassData |= static_cast<unsigned short>(Flags);
}
/// Return the value of this chain of recurrences at the specified
@ -453,7 +455,7 @@ protected:
: SCEVCommutativeExpr(ID, T, O, N) {
assert(isMinMaxType(T));
// Min and max never overflow
setNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW));
setNoWrapFlags(FlagNUW | FlagNSW);
}
public:
@ -539,7 +541,9 @@ class SCEVSequentialMinMaxExpr : public SCEVNAryExpr {
}
/// Set flags for a non-recurrence without clearing previously set flags.
void setNoWrapFlags(NoWrapFlags Flags) { SubclassData |= Flags; }
void setNoWrapFlags(NoWrapFlags Flags) {
SubclassData |= static_cast<unsigned short>(Flags);
}
protected:
/// Note: Constructing subclasses via this constructor is allowed
@ -548,7 +552,7 @@ protected:
: SCEVNAryExpr(ID, T, O, N) {
assert(isSequentialMinMaxType(T));
// Min and max never overflow
setNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW));
setNoWrapFlags(FlagNUW | FlagNSW);
}
public:
@ -1046,6 +1050,15 @@ private:
LoopToScevMapT &Map;
};
template <typename SCEVPtrT>
inline SCEVNoWrapFlags
SCEVUseT<SCEVPtrT>::getNoWrapFlags(SCEVNoWrapFlags Mask) const {
SCEVNoWrapFlags Flags = SCEVNoWrapFlags::FlagAnyWrap;
if (auto *NAry = dyn_cast<SCEVNAryExpr>(Base::getPointer()))
Flags = NAry->getNoWrapFlags();
return (Flags | getUseNoWrapFlags()) & Mask;
}
} // end namespace llvm
#endif // LLVM_ANALYSIS_SCALAREVOLUTIONEXPRESSIONS_H

2
llvm/lib/Analysis/LoopAccessAnalysis.cpp
View File

@ -1028,7 +1028,7 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR,
const DominatorTree &DT,
std::optional<int64_t> Stride = std::nullopt) {
// FIXME: This should probably only return true for NUW.
if (AR->getNoWrapFlags(SCEV::NoWrapMask))
if (any(AR->getNoWrapFlags(SCEV::NoWrapMask)))
return true;
if (Ptr && PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW))

49
llvm/lib/Analysis/ScalarEvolution.cpp
View File

@ -1485,7 +1485,7 @@ static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty,
//
const SCEV *BECount = SE->getBackedgeTakenCount(L);
if (PreAR && PreAR->getNoWrapFlags(WrapType) &&
if (PreAR && any(PreAR->getNoWrapFlags(WrapType)) &&
!isa<SCEVCouldNotCompute>(BECount) && SE->isKnownPositive(BECount))
return PreStart;
@ -1496,7 +1496,7 @@ static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty,
SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
(SE->*GetExtendExpr)(Step, WideTy, Depth));
if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
if (PreAR && AR->getNoWrapFlags(WrapType)) {
if (PreAR && any(AR->getNoWrapFlags(WrapType))) {
// If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
// or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
// `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
@ -1595,7 +1595,7 @@ bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
// Give up if we don't already have the add recurrence we need because
// actually constructing an add recurrence is relatively expensive.
if (PreAR && PreAR->getNoWrapFlags(WrapType)) { // proves (2)
if (PreAR && any(PreAR->getNoWrapFlags(WrapType))) { // proves (2)
const SCEV *DeltaS = getConstant(StartC->getType(), Delta);
ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
@ -1871,8 +1871,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
const SCEV *SResidual =
getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
return getAddExpr(SZExtD, SZExtR,
(SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
return getAddExpr(SZExtD, SZExtR, SCEV::FlagNSW | SCEV::FlagNUW,
Depth + 1);
}
}
@ -1926,8 +1925,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
const SCEV *SResidual =
getAddExpr(getConstant(-D), SA, SCEV::FlagAnyWrap, Depth);
const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
return getAddExpr(SZExtD, SZExtR,
(SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
return getAddExpr(SZExtD, SZExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
Depth + 1);
}
}
@ -2100,8 +2098,7 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
const SCEV *SResidual =
getAddExpr(getConstant(-D), SA, SCEV::FlagAnyWrap, Depth);
const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
return getAddExpr(SSExtD, SSExtR,
(SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
return getAddExpr(SSExtD, SSExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
Depth + 1);
}
}
@ -2224,8 +2221,7 @@ const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
const SCEV *SResidual =
getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
return getAddExpr(SSExtD, SSExtR,
(SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
return getAddExpr(SSExtD, SSExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
Depth + 1);
}
}
@ -2554,7 +2550,7 @@ static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE,
(void)CanAnalyze;
assert(CanAnalyze && "don't call from other places!");
int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
SCEV::NoWrapFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
SCEV::NoWrapFlags SignOrUnsignWrap =
ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
@ -2564,8 +2560,7 @@ static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE,
};
if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Ops, IsKnownNonNegative))
Flags =
ScalarEvolution::setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
Flags = ScalarEvolution::setFlags(Flags, SignOrUnsignMask);
SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
@ -5927,8 +5922,7 @@ const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
if (auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV)) {
setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR),
(SCEV::NoWrapFlags)(AR->getNoWrapFlags() |
proveNoWrapViaConstantRanges(AR)));
(AR->getNoWrapFlags() | proveNoWrapViaConstantRanges(AR)));
}
// We can add Flags to the post-inc expression only if we
@ -6057,9 +6051,9 @@ const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
insertValueToMap(PN, PHISCEV);
if (auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV)) {
setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR),
(SCEV::NoWrapFlags)(AR->getNoWrapFlags() |
proveNoWrapViaConstantRanges(AR)));
setNoWrapFlags(
const_cast<SCEVAddRecExpr *>(AR),
(AR->getNoWrapFlags() | proveNoWrapViaConstantRanges(AR)));
}
// We can add Flags to the post-inc expression only if we
@ -8217,11 +8211,12 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
auto Flags = SCEV::FlagAnyWrap;
if (BO->Op) {
auto MulFlags = getNoWrapFlagsFromUB(BO->Op);
if ((MulFlags & SCEV::FlagNSW) &&
((MulFlags & SCEV::FlagNUW) || SA->getValue().ult(BitWidth - 1)))
Flags = (SCEV::NoWrapFlags)(Flags | SCEV::FlagNSW);
if (MulFlags & SCEV::FlagNUW)
Flags = (SCEV::NoWrapFlags)(Flags | SCEV::FlagNUW);
if (any(MulFlags & SCEV::FlagNSW) &&
(any(MulFlags & SCEV::FlagNUW) ||
SA->getValue().ult(BitWidth - 1)))
Flags = Flags | SCEV::FlagNSW;
if (any(MulFlags & SCEV::FlagNUW))
Flags = Flags | SCEV::FlagNUW;
}
ConstantInt *X = ConstantInt::get(
@ -13362,7 +13357,7 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
// implicit/exceptional) which causes the loop to execute before the
// exiting instruction we're analyzing would trigger UB.
auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
bool NoWrap = ControlsOnlyExit && IV->getNoWrapFlags(WrapType);
bool NoWrap = ControlsOnlyExit && any(IV->getNoWrapFlags(WrapType));
ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
const SCEV *Stride = IV->getStepRecurrence(*this);
@ -13478,7 +13473,7 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
if (!isLoopInvariant(RHS, L)) {
const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(RHS);
if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
RHSAddRec->getNoWrapFlags()) {
any(RHSAddRec->getNoWrapFlags())) {
// The structure of loop we are trying to calculate backedge count of:
//
// left = left_start
@ -13741,7 +13736,7 @@ ScalarEvolution::ExitLimit ScalarEvolution::howManyGreaterThans(
return getCouldNotCompute();
auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
bool NoWrap = ControlsOnlyExit && IV->getNoWrapFlags(WrapType);
bool NoWrap = ControlsOnlyExit && any(IV->getNoWrapFlags(WrapType));
ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
const SCEV *Stride = getNegativeSCEV(IV->getStepRecurrence(*this));

31
llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
View File

@ -302,9 +302,9 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode,
auto canGenerateIncompatiblePoison = [&Flags](Instruction *I) {
// Ensure that no-wrap flags match.
if (isa<OverflowingBinaryOperator>(I)) {
if (I->hasNoSignedWrap() != (Flags & SCEV::FlagNSW))
if (I->hasNoSignedWrap() != any(Flags & SCEV::FlagNSW))
return true;
if (I->hasNoUnsignedWrap() != (Flags & SCEV::FlagNUW))
if (I->hasNoUnsignedWrap() != any(Flags & SCEV::FlagNUW))
return true;
}
// Conservatively, do not use any instruction which has any of exact
@ -341,9 +341,9 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode,
// InstSimplifyFolder.
Instruction *BO = Builder.Insert(BinaryOperator::Create(Opcode, LHS, RHS));
BO->setDebugLoc(Loc);
if (Flags & SCEV::FlagNUW)
if (any(Flags & SCEV::FlagNUW))
BO->setHasNoUnsignedWrap();
if (Flags & SCEV::FlagNSW)
if (any(Flags & SCEV::FlagNSW))
BO->setHasNoSignedWrap();
return BO;
@ -382,8 +382,9 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *Offset, Value *V,
SE.DT.dominates(cast<Instruction>(V), &*Builder.GetInsertPoint()));
Value *Idx = expand(Offset);
GEPNoWrapFlags NW = (Flags & SCEV::FlagNUW) ? GEPNoWrapFlags::noUnsignedWrap()
: GEPNoWrapFlags::none();
GEPNoWrapFlags NW = any(Flags & SCEV::FlagNUW)
? GEPNoWrapFlags::noUnsignedWrap()
: GEPNoWrapFlags::none();
// Fold a GEP with constant operands.
if (Constant *CLHS = dyn_cast<Constant>(V))
@ -573,7 +574,7 @@ Value *SCEVExpander::visitAddExpr(SCEVUseT<const SCEVAddExpr *> S) {
X = SE.getSCEV(U->getValue());
NewOps.push_back(X);
}
Sum = expandAddToGEP(SE.getAddExpr(NewOps), Sum, S->getNoWrapFlags());
Sum = expandAddToGEP(SE.getAddExpr(NewOps), Sum, S.getNoWrapFlags());
} else if (Op->isNonConstantNegative()) {
// Instead of doing a negate and add, just do a subtract.
Value *W = expand(SE.getNegativeSCEV(Op));
@ -586,7 +587,7 @@ Value *SCEVExpander::visitAddExpr(SCEVUseT<const SCEVAddExpr *> S) {
// Canonicalize a constant to the RHS.
if (isa<Constant>(Sum))
std::swap(Sum, W);
Sum = InsertBinop(Instruction::Add, Sum, W, S->getNoWrapFlags(),
Sum = InsertBinop(Instruction::Add, Sum, W, S.getNoWrapFlags(),
/*IsSafeToHoist*/ true);
++I;
}
@ -670,7 +671,7 @@ Value *SCEVExpander::visitMulExpr(SCEVUseT<const SCEVMulExpr *> S) {
if (match(W, m_Power2(RHS))) {
// Canonicalize Prod*(1<<C) to Prod<<C.
assert(!Ty->isVectorTy() && "vector types are not SCEVable");
auto NWFlags = S->getNoWrapFlags();
auto NWFlags = S.getNoWrapFlags();
// clear nsw flag if shl will produce poison value.
if (RHS->logBase2() == RHS->getBitWidth() - 1)
NWFlags = ScalarEvolution::clearFlags(NWFlags, SCEV::FlagNSW);
@ -678,7 +679,7 @@ Value *SCEVExpander::visitMulExpr(SCEVUseT<const SCEVMulExpr *> S) {
ConstantInt::get(Ty, RHS->logBase2()), NWFlags,
/*IsSafeToHoist*/ true);
} else {
Prod = InsertBinop(Instruction::Mul, Prod, W, S->getNoWrapFlags(),
Prod = InsertBinop(Instruction::Mul, Prod, W, S.getNoWrapFlags(),
/*IsSafeToHoist*/ true);
}
}
@ -1340,8 +1341,8 @@ Value *SCEVExpander::visitAddRecExpr(SCEVUseT<const SCEVAddRecExpr *> S) {
SmallVector<SCEVUse, 4> NewOps(S->getNumOperands());
for (unsigned i = 0, e = S->getNumOperands(); i != e; ++i)
NewOps[i] = SE.getAnyExtendExpr(S->getOperand(i), CanonicalIV->getType());
Value *V = expand(SE.getAddRecExpr(NewOps, S->getLoop(),
S->getNoWrapFlags(SCEV::FlagNW)));
Value *V = expand(
SE.getAddRecExpr(NewOps, S->getLoop(), S.getNoWrapFlags(SCEV::FlagNW)));
BasicBlock::iterator NewInsertPt =
findInsertPointAfter(cast<Instruction>(V), &*Builder.GetInsertPoint());
V = expand(SE.getTruncateExpr(SE.getUnknown(V), Ty), NewInsertPt);
@ -1358,13 +1359,13 @@ Value *SCEVExpander::visitAddRecExpr(SCEVUseT<const SCEVAddRecExpr *> S) {
if (isa<PointerType>(S->getType())) {
Value *StartV = expand(SE.getPointerBase(S));
return expandAddToGEP(SE.removePointerBase(S), StartV,
S->getNoWrapFlags(SCEV::FlagNUW));
S.getNoWrapFlags(SCEV::FlagNUW));
}
SmallVector<SCEVUse, 4> NewOps(S->operands());
NewOps[0] = SE.getConstant(Ty, 0);
const SCEV *Rest = SE.getAddRecExpr(NewOps, L,
S->getNoWrapFlags(SCEV::FlagNW));
const SCEV *Rest =
SE.getAddRecExpr(NewOps, L, S.getNoWrapFlags(SCEV::FlagNW));
// Just do a normal add. Pre-expand the operands to suppress folding.
//

49
llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
View File

@ -987,4 +987,53 @@ TEST_F(ScalarEvolutionExpanderTest, GEPFlags) {
EXPECT_EQ(GEP->getNoWrapFlags(), GEPNoWrapFlags::none());
}
// Test that InsertBinop scans existing instructions in the block.
TEST_F(ScalarEvolutionExpanderTest, InsertBinopReuseShlWithMatchingFlags) {
LLVMContext C;
SMDiagnostic Err;
std::unique_ptr<Module> M = parseAssemblyString("define void @f(i64 %n) { "
" ret void "
"}",
Err, C);
assert(M && "Could not parse module?");
assert(!verifyModule(*M) && "Must have been well formed!");
Function *F = M->getFunction("f");
ASSERT_NE(F, nullptr);
ScalarEvolution SE = buildSE(*F);
auto *I64Ty = Type::getInt64Ty(C);
const SCEV *N = SE.getSCEV(F->getArg(0));
const SCEV *Four = SE.getConstant(I64Ty, 4);
const SCEV *Mul = SE.getMulExpr(N, Four, SCEV::FlagNUW | SCEV::FlagNSW);
const SCEV *Expr1 = SE.getAddExpr(Mul, SE.getConstant(I64Ty, 8));
const SCEV *Expr2 = SE.getAddExpr(Mul, SE.getConstant(I64Ty, 16));
// Expand with separate expanders so the InsertedExpressions cache doesn't
// apply.
auto *InsertBefore = F->getEntryBlock().getTerminator();
SCEVExpander Exp1(SE, "expander");
Value *V1 = Exp1.expandCodeFor(Expr1, nullptr, InsertBefore);
SCEVExpander Exp2(SE, "expander");
Value *V2 = Exp2.expandCodeFor(Expr2, nullptr, InsertBefore);
// Both expansions produce different values (different constants added).
EXPECT_NE(V1, V2);
// Both should share the same shl sub-expression via pattern match.
Value *Shl1 = nullptr, *Shl2 = nullptr;
Value *Arg = F->getArg(0);
EXPECT_TRUE(match(V1, m_Add(m_Value(Shl1), m_SpecificInt(8))));
EXPECT_TRUE(match(V2, m_Add(m_Value(Shl2), m_SpecificInt(16))));
EXPECT_EQ(Shl1, Shl2) << "Expected the shl to be reused";
// Verify the shared shl has the expected form and flags.
auto *ShlInst = cast<Instruction>(Shl1);
EXPECT_TRUE(match(ShlInst, m_Shl(m_Specific(Arg), m_SpecificInt(2))));
EXPECT_TRUE(ShlInst->hasNoUnsignedWrap());
EXPECT_TRUE(ShlInst->hasNoSignedWrap());
}
} // end namespace llvm

2
polly/lib/Support/SCEVAffinator.cpp
View File

@ -139,7 +139,7 @@ PWACtx SCEVAffinator::checkForWrapping(const SCEV *Expr, PWACtx PWAC) const {
// whereas n is the number of bits of the Expr, hence:
// n = bitwidth(ExprType)
if (IgnoreIntegerWrapping || (getNoWrapFlags(Expr) & SCEV::FlagNSW))
if (IgnoreIntegerWrapping || any(getNoWrapFlags(Expr) & SCEV::FlagNSW))
return PWAC;
isl::pw_aff PWAMod = addModuloSemantic(PWAC.first, Expr->getType());

3
polly/test/CodeGen/non_affine_float_compare.ll
View File

@ -12,8 +12,7 @@
; CHECK: %[[offset:.*]] = shl nuw nsw i64 %polly.indvar, 2
; CHECK: %scevgep[[R0:[0-9]*]] = getelementptr i8, ptr %A, i64 %[[offset]]
; CHECK: %tmp3_p_scalar_ = load float, ptr %scevgep[[R0]], align 4, !alias.scope !2, !noalias !5
; CHECK: %[[offset2:.*]] = shl nuw nsw i64 %polly.indvar, 2
; CHECK: %scevgep[[R2:[0-9]*]] = getelementptr i8, ptr %scevgep{{[0-9]*}}, i64 %[[offset2]]
; CHECK: %scevgep[[R2:[0-9]*]] = getelementptr i8, ptr %scevgep{{[0-9]*}}, i64 %[[offset]]
; CHECK: %tmp6_p_scalar_ = load float, ptr %scevgep[[R2]], align 4, !alias.scope !2, !noalias !5
; CHECK: %p_tmp7 = fcmp oeq float %tmp3_p_scalar_, %tmp6_p_scalar_
; CHECK: br i1 %p_tmp7, label %polly.stmt.bb8, label %polly.stmt.bb12.[[R:[a-zA-Z_.0-9]*]]