From 29d4fea59bfa8ab3be0773fdd673848ae05cfbfd Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 19 Feb 2026 13:34:46 -0800 Subject: [PATCH] [SLP]Handle mixed select-to-bicasts and general reductions If the reduction tree represents mixed select-to-bitcasts and general reductions, need to handle them correctly to avoid a compiler crash Fixes https://github.com/llvm/llvm-project/pull/181940#issuecomment-3929220929 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 51 ++--- .../Transforms/SLPVectorizer/X86/bool-mask.ll | 177 ++++++++++++++++++ 2 files changed, 207 insertions(+), 21 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 624ca199d4e2..f49803976c0c 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2028,10 +2028,10 @@ public: /// Vectorize the tree but with the list of externally used values \p /// ExternallyUsedValues. Values in this MapVector can be replaced but the /// generated extractvalue instructions. - Value *vectorizeTree( - const ExtraValueToDebugLocsMap &ExternallyUsedValues, - Instruction *ReductionRoot = nullptr, - ArrayRef> VectorValuesAndScales = {}); + Value *vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues, + Instruction *ReductionRoot = nullptr, + ArrayRef> + VectorValuesAndScales = {}); /// \returns the cost incurred by unwanted spills and fills, caused by /// holding live values over call sites. @@ -3584,7 +3584,8 @@ public: template void removeInstructionsAndOperands( ArrayRef DeadVals, - ArrayRef> VectorValuesAndScales) { + ArrayRef> + VectorValuesAndScales) { SmallVector DeadInsts; for (T *V : DeadVals) { auto *I = cast(V); @@ -3652,7 +3653,7 @@ public: if (!DeletedInstructions.contains(OpI) && (!OpI->getType()->isVectorTy() || none_of(VectorValuesAndScales, - [&](const std::tuple &V) { + [&](const std::tuple &V) { return std::get<0>(V) == OpI; })) && isInstructionTriviallyDead(OpI, TLI)) @@ -21680,7 +21681,7 @@ Value *BoUpSLP::vectorizeTree() { Value *BoUpSLP::vectorizeTree( const ExtraValueToDebugLocsMap &ExternallyUsedValues, Instruction *ReductionRoot, - ArrayRef> VectorValuesAndScales) { + ArrayRef> VectorValuesAndScales) { // Clean Entry-to-LastInstruction table. It can be affected after scheduling, // need to rebuild it. EntryToLastInstruction.clear(); @@ -25329,8 +25330,8 @@ class HorizontalReduction { /// The minimum number of the reduced values. const unsigned ReductionLimit = VectorizeNonPowerOf2 ? 3 : 4; /// Contains vector values for reduction including their scale factor and - /// signedness. - SmallVector> VectorValuesAndScales; + /// signedness. The last bool is true, if the value was reduced in-tree. + SmallVector> VectorValuesAndScales; static bool isCmpSelMinMax(Instruction *I) { return match(I, m_Select(m_Cmp(), m_Value(), m_Value())) && @@ -26376,7 +26377,8 @@ public: : 1, RedScalarTy != ScalarTy->getScalarType() ? V.isSignedMinBitwidthRootNode() - : true); + : true, + V.isReducedBitcastRoot() || V.isReducedCmpBitcastRoot()); // Count vectorized reduced values to exclude them from final reduction. for (Value *RdxVal : VL) { @@ -26408,9 +26410,8 @@ public: } if (!VectorValuesAndScales.empty()) VectorizedTree = GetNewVectorizedTree( - VectorizedTree, emitReduction(Builder, *TTI, ReductionRoot->getType(), - V.isReducedBitcastRoot() || - V.isReducedCmpBitcastRoot())); + VectorizedTree, + emitReduction(Builder, *TTI, ReductionRoot->getType())); if (!VectorizedTree) { if (!CheckForReusedReductionOps) { @@ -26820,10 +26821,11 @@ private: /// sub-registers, combines them with the given reduction operation as a /// vector operation and then performs single (small enough) reduction. Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI, - Type *DestTy, bool ReducedInTree) { + Type *DestTy) { Value *ReducedSubTree = nullptr; // Creates reduction and combines with the previous reduction. - auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned) { + auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned, + bool ReducedInTree) { Value *Rdx = createSingleOp(Builder, TTI, Vec, Scale, IsSigned, DestTy, ReducedInTree); if (ReducedSubTree) @@ -26833,15 +26835,21 @@ private: ReducedSubTree = Rdx; }; if (VectorValuesAndScales.size() == 1) { - const auto &[Vec, Scale, IsSigned] = VectorValuesAndScales.front(); - CreateSingleOp(Vec, Scale, IsSigned); + const auto &[Vec, Scale, IsSigned, ReducedInTree] = + VectorValuesAndScales.front(); + CreateSingleOp(Vec, Scale, IsSigned, ReducedInTree); return ReducedSubTree; } // Scales Vec using given Cnt scale factor and then performs vector combine // with previous value of VecOp. Value *VecRes = nullptr; bool VecResSignedness = false; - auto CreateVecOp = [&](Value *Vec, unsigned Cnt, bool IsSigned) { + auto CreateVecOp = [&](Value *Vec, unsigned Cnt, bool IsSigned, + bool ReducedInTree) { + if (ReducedInTree) { + CreateSingleOp(Vec, Cnt, IsSigned, ReducedInTree); + return; + } Type *ScalarTy = Vec->getType()->getScalarType(); // Scale Vec using given Cnt scale factor. if (Cnt > 1) { @@ -26970,9 +26978,10 @@ private: VecRes = Op; } }; - for (auto [Vec, Scale, IsSigned] : VectorValuesAndScales) - CreateVecOp(Vec, Scale, IsSigned); - CreateSingleOp(VecRes, /*Scale=*/1, /*IsSigned=*/false); + for (auto [Vec, Scale, IsSigned, ReducedInTree] : VectorValuesAndScales) + CreateVecOp(Vec, Scale, IsSigned, ReducedInTree); + CreateSingleOp(VecRes, /*Scale=*/1, /*IsSigned=*/false, + /*ReducedInTree=*/false); return ReducedSubTree; } diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll index 9073600eb380..2780dd363035 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll @@ -363,3 +363,180 @@ entry: %mask.1.7 = or i64 %or.7, %mask.1.6 ret i64 %mask.1.7 } + +define i64 @combined(ptr nocapture noundef readonly %src) { +; SSE2-LABEL: @combined( +; SSE2-NEXT: entry: +; SSE2-NEXT: [[TMP0:%.*]] = load i64, ptr [[SRC:%.*]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i64 [[TMP0]], 0 +; SSE2-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64 +; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 1 +; SSE2-NEXT: [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX_1]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_1:%.*]] = icmp eq i64 [[TMP1]], 0 +; SSE2-NEXT: [[OR_1:%.*]] = select i1 [[TOBOOL_NOT_1]], i64 0, i64 2 +; SSE2-NEXT: [[MASK_1_1:%.*]] = or i64 [[OR_1]], [[OR]] +; SSE2-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 2 +; SSE2-NEXT: [[TMP2:%.*]] = load i64, ptr [[ARRAYIDX_2]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_2:%.*]] = icmp eq i64 [[TMP2]], 0 +; SSE2-NEXT: [[OR_2:%.*]] = select i1 [[TOBOOL_NOT_2]], i64 0, i64 4 +; SSE2-NEXT: [[MASK_1_2:%.*]] = or i64 [[OR_2]], [[MASK_1_1]] +; SSE2-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 3 +; SSE2-NEXT: [[TMP3:%.*]] = load i64, ptr [[ARRAYIDX_3]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_3:%.*]] = icmp eq i64 [[TMP3]], 0 +; SSE2-NEXT: [[OR_3:%.*]] = select i1 [[TOBOOL_NOT_3]], i64 0, i64 8 +; SSE2-NEXT: [[MASK_1_3:%.*]] = or i64 [[OR_3]], [[MASK_1_2]] +; SSE2-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 4 +; SSE2-NEXT: [[TMP4:%.*]] = load i64, ptr [[ARRAYIDX_4]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_4:%.*]] = icmp eq i64 [[TMP4]], 0 +; SSE2-NEXT: [[OR_4:%.*]] = select i1 [[TOBOOL_NOT_4]], i64 0, i64 16 +; SSE2-NEXT: [[MASK_1_4:%.*]] = or i64 [[OR_4]], [[MASK_1_3]] +; SSE2-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 5 +; SSE2-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX_5]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i64 [[TMP5]], 0 +; SSE2-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32 +; SSE2-NEXT: [[MASK_1_5:%.*]] = or i64 [[OR_5]], [[MASK_1_4]] +; SSE2-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 6 +; SSE2-NEXT: [[TMP6:%.*]] = load i64, ptr [[ARRAYIDX_6]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i64 [[TMP6]], 0 +; SSE2-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64 +; SSE2-NEXT: [[MASK_1_6:%.*]] = or i64 [[OR_6]], [[MASK_1_5]] +; SSE2-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 7 +; SSE2-NEXT: [[TMP7:%.*]] = load i64, ptr [[ARRAYIDX_7]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i64 [[TMP7]], 0 +; SSE2-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128 +; SSE2-NEXT: [[MASK_1_7:%.*]] = or i64 [[OR_7]], [[MASK_1_6]] +; SSE2-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 8 +; SSE2-NEXT: [[TMP8:%.*]] = load i64, ptr [[ARRAYIDX_8]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_8:%.*]] = icmp eq i64 [[TMP8]], 0 +; SSE2-NEXT: [[OR_8:%.*]] = select i1 [[TOBOOL_NOT_8]], i64 0, i64 [[TMP8]] +; SSE2-NEXT: [[MASK_1_8:%.*]] = or i64 [[OR_8]], [[MASK_1_7]] +; SSE2-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 9 +; SSE2-NEXT: [[TMP9:%.*]] = load i64, ptr [[ARRAYIDX_9]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_9:%.*]] = icmp eq i64 [[TMP9]], 0 +; SSE2-NEXT: [[OR_9:%.*]] = select i1 [[TOBOOL_NOT_9]], i64 0, i64 [[TMP9]] +; SSE2-NEXT: [[MASK_1_9:%.*]] = or i64 [[OR_9]], [[MASK_1_8]] +; SSE2-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 10 +; SSE2-NEXT: [[TMP10:%.*]] = load i64, ptr [[ARRAYIDX_10]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_10:%.*]] = icmp eq i64 [[TMP10]], 0 +; SSE2-NEXT: [[OR_10:%.*]] = select i1 [[TOBOOL_NOT_10]], i64 0, i64 [[TMP10]] +; SSE2-NEXT: [[MASK_1_10:%.*]] = or i64 [[OR_10]], [[MASK_1_9]] +; SSE2-NEXT: [[ARRAYIDX_11:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 11 +; SSE2-NEXT: [[TMP11:%.*]] = load i64, ptr [[ARRAYIDX_11]], align 2 +; SSE2-NEXT: [[TOBOOL_NOT_11:%.*]] = icmp eq i64 [[TMP11]], 0 +; SSE2-NEXT: [[OR_11:%.*]] = select i1 [[TOBOOL_NOT_11]], i64 0, i64 [[TMP10]] +; SSE2-NEXT: [[MASK_1_11:%.*]] = or i64 [[OR_11]], [[MASK_1_10]] +; SSE2-NEXT: ret i64 [[MASK_1_11]] +; +; SSE4-LABEL: @combined( +; SSE4-NEXT: entry: +; SSE4-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 2 +; SSE4-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[ARRAYIDX_2]], align 2 +; SSE4-NEXT: [[TMP1:%.*]] = icmp eq <8 x i64> [[TMP0]], zeroinitializer +; SSE4-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[TMP0]], <8 x i64> , <8 x i32> +; SSE4-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> zeroinitializer, <8 x i64> [[TMP2]] +; SSE4-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 10 +; SSE4-NEXT: [[TMP4:%.*]] = load <2 x i64>, ptr [[SRC]], align 2 +; SSE4-NEXT: [[TMP5:%.*]] = load <2 x i64>, ptr [[ARRAYIDX_10]], align 2 +; SSE4-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> poison, <4 x i32> +; SSE4-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> poison, <4 x i32> +; SSE4-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <4 x i32> +; SSE4-NEXT: [[TMP9:%.*]] = icmp ne <4 x i64> [[TMP8]], zeroinitializer +; SSE4-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> , <4 x i32> +; SSE4-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP9]], <4 x i64> [[TMP10]], <4 x i64> zeroinitializer +; SSE4-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP3]]) +; SSE4-NEXT: [[TMP13:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP11]]) +; SSE4-NEXT: [[OP_RDX4:%.*]] = or i64 [[TMP13]], [[TMP12]] +; SSE4-NEXT: ret i64 [[OP_RDX4]] +; +; AVX-LABEL: @combined( +; AVX-NEXT: entry: +; AVX-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 2 +; AVX-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer +; AVX-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8 +; AVX-NEXT: [[TMP3:%.*]] = zext i8 [[TMP2]] to i64 +; AVX-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 8 +; AVX-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[ARRAYIDX_8]], align 2 +; AVX-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], zeroinitializer +; AVX-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> +; AVX-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> zeroinitializer, <4 x i64> [[TMP6]] +; AVX-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP7]]) +; AVX-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP3]], [[TMP8]] +; AVX-NEXT: ret i64 [[OP_RDX]] +; +; AVX512-LABEL: @combined( +; AVX512-NEXT: entry: +; AVX512-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 2 +; AVX512-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer +; AVX512-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8 +; AVX512-NEXT: [[TMP3:%.*]] = zext i8 [[TMP2]] to i64 +; AVX512-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 8 +; AVX512-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[ARRAYIDX_8]], align 2 +; AVX512-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], zeroinitializer +; AVX512-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> +; AVX512-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> zeroinitializer, <4 x i64> [[TMP6]] +; AVX512-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP7]]) +; AVX512-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP3]], [[TMP8]] +; AVX512-NEXT: ret i64 [[OP_RDX]] +; +entry: + %0 = load i64, ptr %src, align 2 + %tobool.not = icmp ne i64 %0, 0 + %or = zext i1 %tobool.not to i64 + %arrayidx.1 = getelementptr inbounds i64, ptr %src, i64 1 + %1 = load i64, ptr %arrayidx.1, align 2 + %tobool.not.1 = icmp eq i64 %1, 0 + %or.1 = select i1 %tobool.not.1, i64 0, i64 2 + %mask.1.1 = or i64 %or.1, %or + %arrayidx.2 = getelementptr inbounds i64, ptr %src, i64 2 + %2 = load i64, ptr %arrayidx.2, align 2 + %tobool.not.2 = icmp eq i64 %2, 0 + %or.2 = select i1 %tobool.not.2, i64 0, i64 4 + %mask.1.2 = or i64 %or.2, %mask.1.1 + %arrayidx.3 = getelementptr inbounds i64, ptr %src, i64 3 + %3 = load i64, ptr %arrayidx.3, align 2 + %tobool.not.3 = icmp eq i64 %3, 0 + %or.3 = select i1 %tobool.not.3, i64 0, i64 8 + %mask.1.3 = or i64 %or.3, %mask.1.2 + %arrayidx.4 = getelementptr inbounds i64, ptr %src, i64 4 + %4 = load i64, ptr %arrayidx.4, align 2 + %tobool.not.4 = icmp eq i64 %4, 0 + %or.4 = select i1 %tobool.not.4, i64 0, i64 16 + %mask.1.4 = or i64 %or.4, %mask.1.3 + %arrayidx.5 = getelementptr inbounds i64, ptr %src, i64 5 + %5 = load i64, ptr %arrayidx.5, align 2 + %tobool.not.5 = icmp eq i64 %5, 0 + %or.5 = select i1 %tobool.not.5, i64 0, i64 32 + %mask.1.5 = or i64 %or.5, %mask.1.4 + %arrayidx.6 = getelementptr inbounds i64, ptr %src, i64 6 + %6 = load i64, ptr %arrayidx.6, align 2 + %tobool.not.6 = icmp eq i64 %6, 0 + %or.6 = select i1 %tobool.not.6, i64 0, i64 64 + %mask.1.6 = or i64 %or.6, %mask.1.5 + %arrayidx.7 = getelementptr inbounds i64, ptr %src, i64 7 + %7 = load i64, ptr %arrayidx.7, align 2 + %tobool.not.7 = icmp eq i64 %7, 0 + %or.7 = select i1 %tobool.not.7, i64 0, i64 128 + %mask.1.7 = or i64 %or.7, %mask.1.6 + %arrayidx.8 = getelementptr inbounds i64, ptr %src, i64 8 + %8 = load i64, ptr %arrayidx.8, align 2 + %tobool.not.8 = icmp eq i64 %8, 0 + %or.8 = select i1 %tobool.not.8, i64 0, i64 %8 + %mask.1.8 = or i64 %or.8, %mask.1.7 + %arrayidx.9 = getelementptr inbounds i64, ptr %src, i64 9 + %9 = load i64, ptr %arrayidx.9, align 2 + %tobool.not.9 = icmp eq i64 %9, 0 + %or.9 = select i1 %tobool.not.9, i64 0, i64 %9 + %mask.1.9 = or i64 %or.9, %mask.1.8 + %arrayidx.10 = getelementptr inbounds i64, ptr %src, i64 10 + %10 = load i64, ptr %arrayidx.10, align 2 + %tobool.not.10 = icmp eq i64 %10, 0 + %or.10 = select i1 %tobool.not.10, i64 0, i64 %10 + %mask.1.10 = or i64 %or.10, %mask.1.9 + %arrayidx.11 = getelementptr inbounds i64, ptr %src, i64 11 + %11 = load i64, ptr %arrayidx.11, align 2 + %tobool.not.11 = icmp eq i64 %11, 0 + %or.11 = select i1 %tobool.not.11, i64 0, i64 %10 + %mask.1.11 = or i64 %or.11, %mask.1.10 + ret i64 %mask.1.11 +} +