[SLP]Handle mixed select-to-bicasts and general reductions

If the reduction tree represents mixed select-to-bitcasts and general
reductions, need to handle them correctly to avoid a compiler crash

Fixes https://github.com/llvm/llvm-project/pull/181940#issuecomment-3929220929
This commit is contained in:
Alexey Bataev 2026-02-19 13:34:46 -08:00
parent 650e898c45
commit 29d4fea59b
2 changed files with 207 additions and 21 deletions

View File

@ -2028,10 +2028,10 @@ public:
/// Vectorize the tree but with the list of externally used values \p
/// ExternallyUsedValues. Values in this MapVector can be replaced but the
/// generated extractvalue instructions.
Value *vectorizeTree(
const ExtraValueToDebugLocsMap &ExternallyUsedValues,
Instruction *ReductionRoot = nullptr,
ArrayRef<std::tuple<Value *, unsigned, bool>> VectorValuesAndScales = {});
Value *vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
Instruction *ReductionRoot = nullptr,
ArrayRef<std::tuple<Value *, unsigned, bool, bool>>
VectorValuesAndScales = {});
/// \returns the cost incurred by unwanted spills and fills, caused by
/// holding live values over call sites.
@ -3584,7 +3584,8 @@ public:
template <typename T>
void removeInstructionsAndOperands(
ArrayRef<T *> DeadVals,
ArrayRef<std::tuple<Value *, unsigned, bool>> VectorValuesAndScales) {
ArrayRef<std::tuple<Value *, unsigned, bool, bool>>
VectorValuesAndScales) {
SmallVector<WeakTrackingVH> DeadInsts;
for (T *V : DeadVals) {
auto *I = cast<Instruction>(V);
@ -3652,7 +3653,7 @@ public:
if (!DeletedInstructions.contains(OpI) &&
(!OpI->getType()->isVectorTy() ||
none_of(VectorValuesAndScales,
[&](const std::tuple<Value *, unsigned, bool> &V) {
[&](const std::tuple<Value *, unsigned, bool, bool> &V) {
return std::get<0>(V) == OpI;
})) &&
isInstructionTriviallyDead(OpI, TLI))
@ -21680,7 +21681,7 @@ Value *BoUpSLP::vectorizeTree() {
Value *BoUpSLP::vectorizeTree(
const ExtraValueToDebugLocsMap &ExternallyUsedValues,
Instruction *ReductionRoot,
ArrayRef<std::tuple<Value *, unsigned, bool>> VectorValuesAndScales) {
ArrayRef<std::tuple<Value *, unsigned, bool, bool>> VectorValuesAndScales) {
// Clean Entry-to-LastInstruction table. It can be affected after scheduling,
// need to rebuild it.
EntryToLastInstruction.clear();
@ -25329,8 +25330,8 @@ class HorizontalReduction {
/// The minimum number of the reduced values.
const unsigned ReductionLimit = VectorizeNonPowerOf2 ? 3 : 4;
/// Contains vector values for reduction including their scale factor and
/// signedness.
SmallVector<std::tuple<Value *, unsigned, bool>> VectorValuesAndScales;
/// signedness. The last bool is true, if the value was reduced in-tree.
SmallVector<std::tuple<Value *, unsigned, bool, bool>> VectorValuesAndScales;
static bool isCmpSelMinMax(Instruction *I) {
return match(I, m_Select(m_Cmp(), m_Value(), m_Value())) &&
@ -26376,7 +26377,8 @@ public:
: 1,
RedScalarTy != ScalarTy->getScalarType()
? V.isSignedMinBitwidthRootNode()
: true);
: true,
V.isReducedBitcastRoot() || V.isReducedCmpBitcastRoot());
// Count vectorized reduced values to exclude them from final reduction.
for (Value *RdxVal : VL) {
@ -26408,9 +26410,8 @@ public:
}
if (!VectorValuesAndScales.empty())
VectorizedTree = GetNewVectorizedTree(
VectorizedTree, emitReduction(Builder, *TTI, ReductionRoot->getType(),
V.isReducedBitcastRoot() ||
V.isReducedCmpBitcastRoot()));
VectorizedTree,
emitReduction(Builder, *TTI, ReductionRoot->getType()));
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
@ -26820,10 +26821,11 @@ private:
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
Type *DestTy, bool ReducedInTree) {
Type *DestTy) {
Value *ReducedSubTree = nullptr;
// Creates reduction and combines with the previous reduction.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned) {
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
bool ReducedInTree) {
Value *Rdx = createSingleOp(Builder, TTI, Vec, Scale, IsSigned, DestTy,
ReducedInTree);
if (ReducedSubTree)
@ -26833,15 +26835,21 @@ private:
ReducedSubTree = Rdx;
};
if (VectorValuesAndScales.size() == 1) {
const auto &[Vec, Scale, IsSigned] = VectorValuesAndScales.front();
CreateSingleOp(Vec, Scale, IsSigned);
const auto &[Vec, Scale, IsSigned, ReducedInTree] =
VectorValuesAndScales.front();
CreateSingleOp(Vec, Scale, IsSigned, ReducedInTree);
return ReducedSubTree;
}
// Scales Vec using given Cnt scale factor and then performs vector combine
// with previous value of VecOp.
Value *VecRes = nullptr;
bool VecResSignedness = false;
auto CreateVecOp = [&](Value *Vec, unsigned Cnt, bool IsSigned) {
auto CreateVecOp = [&](Value *Vec, unsigned Cnt, bool IsSigned,
bool ReducedInTree) {
if (ReducedInTree) {
CreateSingleOp(Vec, Cnt, IsSigned, ReducedInTree);
return;
}
Type *ScalarTy = Vec->getType()->getScalarType();
// Scale Vec using given Cnt scale factor.
if (Cnt > 1) {
@ -26970,9 +26978,10 @@ private:
VecRes = Op;
}
};
for (auto [Vec, Scale, IsSigned] : VectorValuesAndScales)
CreateVecOp(Vec, Scale, IsSigned);
CreateSingleOp(VecRes, /*Scale=*/1, /*IsSigned=*/false);
for (auto [Vec, Scale, IsSigned, ReducedInTree] : VectorValuesAndScales)
CreateVecOp(Vec, Scale, IsSigned, ReducedInTree);
CreateSingleOp(VecRes, /*Scale=*/1, /*IsSigned=*/false,
/*ReducedInTree=*/false);
return ReducedSubTree;
}

View File

@ -363,3 +363,180 @@ entry:
%mask.1.7 = or i64 %or.7, %mask.1.6
ret i64 %mask.1.7
}
define i64 @combined(ptr nocapture noundef readonly %src) {
; SSE2-LABEL: @combined(
; SSE2-NEXT: entry:
; SSE2-NEXT: [[TMP0:%.*]] = load i64, ptr [[SRC:%.*]], align 2
; SSE2-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i64 [[TMP0]], 0
; SSE2-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64
; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 1
; SSE2-NEXT: [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX_1]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_1:%.*]] = icmp eq i64 [[TMP1]], 0
; SSE2-NEXT: [[OR_1:%.*]] = select i1 [[TOBOOL_NOT_1]], i64 0, i64 2
; SSE2-NEXT: [[MASK_1_1:%.*]] = or i64 [[OR_1]], [[OR]]
; SSE2-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 2
; SSE2-NEXT: [[TMP2:%.*]] = load i64, ptr [[ARRAYIDX_2]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_2:%.*]] = icmp eq i64 [[TMP2]], 0
; SSE2-NEXT: [[OR_2:%.*]] = select i1 [[TOBOOL_NOT_2]], i64 0, i64 4
; SSE2-NEXT: [[MASK_1_2:%.*]] = or i64 [[OR_2]], [[MASK_1_1]]
; SSE2-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 3
; SSE2-NEXT: [[TMP3:%.*]] = load i64, ptr [[ARRAYIDX_3]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_3:%.*]] = icmp eq i64 [[TMP3]], 0
; SSE2-NEXT: [[OR_3:%.*]] = select i1 [[TOBOOL_NOT_3]], i64 0, i64 8
; SSE2-NEXT: [[MASK_1_3:%.*]] = or i64 [[OR_3]], [[MASK_1_2]]
; SSE2-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 4
; SSE2-NEXT: [[TMP4:%.*]] = load i64, ptr [[ARRAYIDX_4]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_4:%.*]] = icmp eq i64 [[TMP4]], 0
; SSE2-NEXT: [[OR_4:%.*]] = select i1 [[TOBOOL_NOT_4]], i64 0, i64 16
; SSE2-NEXT: [[MASK_1_4:%.*]] = or i64 [[OR_4]], [[MASK_1_3]]
; SSE2-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 5
; SSE2-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX_5]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i64 [[TMP5]], 0
; SSE2-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32
; SSE2-NEXT: [[MASK_1_5:%.*]] = or i64 [[OR_5]], [[MASK_1_4]]
; SSE2-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 6
; SSE2-NEXT: [[TMP6:%.*]] = load i64, ptr [[ARRAYIDX_6]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i64 [[TMP6]], 0
; SSE2-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64
; SSE2-NEXT: [[MASK_1_6:%.*]] = or i64 [[OR_6]], [[MASK_1_5]]
; SSE2-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 7
; SSE2-NEXT: [[TMP7:%.*]] = load i64, ptr [[ARRAYIDX_7]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i64 [[TMP7]], 0
; SSE2-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128
; SSE2-NEXT: [[MASK_1_7:%.*]] = or i64 [[OR_7]], [[MASK_1_6]]
; SSE2-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 8
; SSE2-NEXT: [[TMP8:%.*]] = load i64, ptr [[ARRAYIDX_8]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_8:%.*]] = icmp eq i64 [[TMP8]], 0
; SSE2-NEXT: [[OR_8:%.*]] = select i1 [[TOBOOL_NOT_8]], i64 0, i64 [[TMP8]]
; SSE2-NEXT: [[MASK_1_8:%.*]] = or i64 [[OR_8]], [[MASK_1_7]]
; SSE2-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 9
; SSE2-NEXT: [[TMP9:%.*]] = load i64, ptr [[ARRAYIDX_9]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_9:%.*]] = icmp eq i64 [[TMP9]], 0
; SSE2-NEXT: [[OR_9:%.*]] = select i1 [[TOBOOL_NOT_9]], i64 0, i64 [[TMP9]]
; SSE2-NEXT: [[MASK_1_9:%.*]] = or i64 [[OR_9]], [[MASK_1_8]]
; SSE2-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 10
; SSE2-NEXT: [[TMP10:%.*]] = load i64, ptr [[ARRAYIDX_10]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_10:%.*]] = icmp eq i64 [[TMP10]], 0
; SSE2-NEXT: [[OR_10:%.*]] = select i1 [[TOBOOL_NOT_10]], i64 0, i64 [[TMP10]]
; SSE2-NEXT: [[MASK_1_10:%.*]] = or i64 [[OR_10]], [[MASK_1_9]]
; SSE2-NEXT: [[ARRAYIDX_11:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 11
; SSE2-NEXT: [[TMP11:%.*]] = load i64, ptr [[ARRAYIDX_11]], align 2
; SSE2-NEXT: [[TOBOOL_NOT_11:%.*]] = icmp eq i64 [[TMP11]], 0
; SSE2-NEXT: [[OR_11:%.*]] = select i1 [[TOBOOL_NOT_11]], i64 0, i64 [[TMP10]]
; SSE2-NEXT: [[MASK_1_11:%.*]] = or i64 [[OR_11]], [[MASK_1_10]]
; SSE2-NEXT: ret i64 [[MASK_1_11]]
;
; SSE4-LABEL: @combined(
; SSE4-NEXT: entry:
; SSE4-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 2
; SSE4-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[ARRAYIDX_2]], align 2
; SSE4-NEXT: [[TMP1:%.*]] = icmp eq <8 x i64> [[TMP0]], zeroinitializer
; SSE4-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[TMP0]], <8 x i64> <i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 poison, i64 poison>, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 6, i32 7>
; SSE4-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]]
; SSE4-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 10
; SSE4-NEXT: [[TMP4:%.*]] = load <2 x i64>, ptr [[SRC]], align 2
; SSE4-NEXT: [[TMP5:%.*]] = load <2 x i64>, ptr [[ARRAYIDX_10]], align 2
; SSE4-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; SSE4-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; SSE4-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; SSE4-NEXT: [[TMP9:%.*]] = icmp ne <4 x i64> [[TMP8]], zeroinitializer
; SSE4-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> <i64 1, i64 2, i64 poison, i64 poison>, <4 x i32> <i32 4, i32 5, i32 2, i32 2>
; SSE4-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> [[TMP10]], <4 x i64> zeroinitializer
; SSE4-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP3]])
; SSE4-NEXT: [[TMP13:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP11]])
; SSE4-NEXT: [[OP_RDX4:%.*]] = or i64 [[TMP13]], [[TMP12]]
; SSE4-NEXT: ret i64 [[OP_RDX4]]
;
; AVX-LABEL: @combined(
; AVX-NEXT: entry:
; AVX-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 2
; AVX-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer
; AVX-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
; AVX-NEXT: [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
; AVX-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 8
; AVX-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[ARRAYIDX_8]], align 2
; AVX-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], zeroinitializer
; AVX-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; AVX-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> zeroinitializer, <4 x i64> [[TMP6]]
; AVX-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP7]])
; AVX-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP3]], [[TMP8]]
; AVX-NEXT: ret i64 [[OP_RDX]]
;
; AVX512-LABEL: @combined(
; AVX512-NEXT: entry:
; AVX512-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 2
; AVX512-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer
; AVX512-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
; AVX512-NEXT: [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
; AVX512-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 8
; AVX512-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[ARRAYIDX_8]], align 2
; AVX512-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], zeroinitializer
; AVX512-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; AVX512-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> zeroinitializer, <4 x i64> [[TMP6]]
; AVX512-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP7]])
; AVX512-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP3]], [[TMP8]]
; AVX512-NEXT: ret i64 [[OP_RDX]]
;
entry:
%0 = load i64, ptr %src, align 2
%tobool.not = icmp ne i64 %0, 0
%or = zext i1 %tobool.not to i64
%arrayidx.1 = getelementptr inbounds i64, ptr %src, i64 1
%1 = load i64, ptr %arrayidx.1, align 2
%tobool.not.1 = icmp eq i64 %1, 0
%or.1 = select i1 %tobool.not.1, i64 0, i64 2
%mask.1.1 = or i64 %or.1, %or
%arrayidx.2 = getelementptr inbounds i64, ptr %src, i64 2
%2 = load i64, ptr %arrayidx.2, align 2
%tobool.not.2 = icmp eq i64 %2, 0
%or.2 = select i1 %tobool.not.2, i64 0, i64 4
%mask.1.2 = or i64 %or.2, %mask.1.1
%arrayidx.3 = getelementptr inbounds i64, ptr %src, i64 3
%3 = load i64, ptr %arrayidx.3, align 2
%tobool.not.3 = icmp eq i64 %3, 0
%or.3 = select i1 %tobool.not.3, i64 0, i64 8
%mask.1.3 = or i64 %or.3, %mask.1.2
%arrayidx.4 = getelementptr inbounds i64, ptr %src, i64 4
%4 = load i64, ptr %arrayidx.4, align 2
%tobool.not.4 = icmp eq i64 %4, 0
%or.4 = select i1 %tobool.not.4, i64 0, i64 16
%mask.1.4 = or i64 %or.4, %mask.1.3
%arrayidx.5 = getelementptr inbounds i64, ptr %src, i64 5
%5 = load i64, ptr %arrayidx.5, align 2
%tobool.not.5 = icmp eq i64 %5, 0
%or.5 = select i1 %tobool.not.5, i64 0, i64 32
%mask.1.5 = or i64 %or.5, %mask.1.4
%arrayidx.6 = getelementptr inbounds i64, ptr %src, i64 6
%6 = load i64, ptr %arrayidx.6, align 2
%tobool.not.6 = icmp eq i64 %6, 0
%or.6 = select i1 %tobool.not.6, i64 0, i64 64
%mask.1.6 = or i64 %or.6, %mask.1.5
%arrayidx.7 = getelementptr inbounds i64, ptr %src, i64 7
%7 = load i64, ptr %arrayidx.7, align 2
%tobool.not.7 = icmp eq i64 %7, 0
%or.7 = select i1 %tobool.not.7, i64 0, i64 128
%mask.1.7 = or i64 %or.7, %mask.1.6
%arrayidx.8 = getelementptr inbounds i64, ptr %src, i64 8
%8 = load i64, ptr %arrayidx.8, align 2
%tobool.not.8 = icmp eq i64 %8, 0
%or.8 = select i1 %tobool.not.8, i64 0, i64 %8
%mask.1.8 = or i64 %or.8, %mask.1.7
%arrayidx.9 = getelementptr inbounds i64, ptr %src, i64 9
%9 = load i64, ptr %arrayidx.9, align 2
%tobool.not.9 = icmp eq i64 %9, 0
%or.9 = select i1 %tobool.not.9, i64 0, i64 %9
%mask.1.9 = or i64 %or.9, %mask.1.8
%arrayidx.10 = getelementptr inbounds i64, ptr %src, i64 10
%10 = load i64, ptr %arrayidx.10, align 2
%tobool.not.10 = icmp eq i64 %10, 0
%or.10 = select i1 %tobool.not.10, i64 0, i64 %10
%mask.1.10 = or i64 %or.10, %mask.1.9
%arrayidx.11 = getelementptr inbounds i64, ptr %src, i64 11
%11 = load i64, ptr %arrayidx.11, align 2
%tobool.not.11 = icmp eq i64 %11, 0
%or.11 = select i1 %tobool.not.11, i64 0, i64 %10
%mask.1.11 = or i64 %or.11, %mask.1.10
ret i64 %mask.1.11
}