diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index ecfcf64c7991..99a65265075a 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -324,36 +324,33 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, return LT.first * getLMULCost(LT.second); } } - - // vrgather + cost of generating the mask constant. - // We model this for an unknown mask with a single vrgather. - if (LT.first == 1 && - (LT.second.getScalarSizeInBits() != 8 || - LT.second.getVectorNumElements() <= 256)) { - VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, Tp->getContext()); - InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); - return IndexCost + getVRGatherVVCost(LT.second); - } + } + // vrgather + cost of generating the mask constant. + // We model this for an unknown mask with a single vrgather. + if (LT.second.isFixedLengthVector() && LT.first == 1 && + (LT.second.getScalarSizeInBits() != 8 || + LT.second.getVectorNumElements() <= 256)) { + VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, Tp->getContext()); + InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); + return IndexCost + getVRGatherVVCost(LT.second); } [[fallthrough]]; } case TTI::SK_Transpose: case TTI::SK_PermuteTwoSrc: { - if (Mask.size() >= 2 && LT.second.isFixedLengthVector()) { - // 2 x (vrgather + cost of generating the mask constant) + cost of mask - // register for the second vrgather. We model this for an unknown - // (shuffle) mask. - if (LT.first == 1 && - (LT.second.getScalarSizeInBits() != 8 || - LT.second.getVectorNumElements() <= 256)) { - auto &C = Tp->getContext(); - auto EC = Tp->getElementCount(); - VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C); - VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC); - InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); - InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind); - return 2 * IndexCost + 2 * getVRGatherVVCost(LT.second) + MaskCost; - } + // 2 x (vrgather + cost of generating the mask constant) + cost of mask + // register for the second vrgather. We model this for an unknown + // (shuffle) mask. + if (LT.second.isFixedLengthVector() && LT.first == 1 && + (LT.second.getScalarSizeInBits() != 8 || + LT.second.getVectorNumElements() <= 256)) { + auto &C = Tp->getContext(); + auto EC = Tp->getElementCount(); + VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C); + VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC); + InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); + InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind); + return 2 * IndexCost + 2 * getVRGatherVVCost(LT.second) + MaskCost; } [[fallthrough]]; } diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll index d6921f6a6eca..35908ee72bc2 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll @@ -1,13 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \ -; RUN: -riscv-v-vector-bits-min=128 -riscv-v-slp-max-vf=0 -S \ -; RUN: | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \ -; RUN: -riscv-v-vector-bits-min=256 -riscv-v-slp-max-vf=0 -S \ -; RUN: | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \ -; RUN: -riscv-v-vector-bits-min=512 -riscv-v-slp-max-vf=0 -S \ -; RUN: | FileCheck %s --check-prefixes=CHECK +; RUN: -riscv-v-slp-max-vf=0 -S \ +; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL128 +; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v,+zvl256b \ +; RUN: -riscv-v-slp-max-vf=0 -S \ +; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL256 +; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v,+zvl512b \ +; RUN: -riscv-v-slp-max-vf=0 -S \ +; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL512 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128" target triple = "riscv64" @@ -274,20 +274,47 @@ entry: } define void @reduce_or_2() { -; CHECK-LABEL: @reduce_or_2( -; CHECK-NEXT: [[TMP1:%.*]] = shl i64 0, 0 -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 15 -; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 6 -; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]]) -; CHECK-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]] -; CHECK-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]] -; CHECK: 8: -; CHECK-NEXT: ret void -; CHECK: 9: -; CHECK-NEXT: ret void +; ZVL128-LABEL: @reduce_or_2( +; ZVL128-NEXT: [[TMP1:%.*]] = shl i64 0, 0 +; ZVL128-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 15 +; ZVL128-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer +; ZVL128-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 6 +; ZVL128-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer +; ZVL128-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; ZVL128-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]]) +; ZVL128-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]] +; ZVL128-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]] +; ZVL128: 8: +; ZVL128-NEXT: ret void +; ZVL128: 9: +; ZVL128-NEXT: ret void +; +; ZVL256-LABEL: @reduce_or_2( +; ZVL256-NEXT: [[TMP1:%.*]] = shl i64 0, 0 +; ZVL256-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 15 +; ZVL256-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer +; ZVL256-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 6 +; ZVL256-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer +; ZVL256-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; ZVL256-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]]) +; ZVL256-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]] +; ZVL256-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]] +; ZVL256: 8: +; ZVL256-NEXT: ret void +; ZVL256: 9: +; ZVL256-NEXT: ret void +; +; ZVL512-LABEL: @reduce_or_2( +; ZVL512-NEXT: [[TMP1:%.*]] = shl i64 0, 0 +; ZVL512-NEXT: [[TMP2:%.*]] = insertelement <32 x i64> , i64 [[TMP1]], i32 15 +; ZVL512-NEXT: [[TMP3:%.*]] = shufflevector <32 x i64> [[TMP2]], <32 x i64> poison, <32 x i32> +; ZVL512-NEXT: [[TMP4:%.*]] = icmp ult <32 x i64> [[TMP3]], zeroinitializer +; ZVL512-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> [[TMP4]]) +; ZVL512-NEXT: br i1 [[TMP5]], label [[TMP7:%.*]], label [[TMP6:%.*]] +; ZVL512: 6: +; ZVL512-NEXT: ret void +; ZVL512: 7: +; ZVL512-NEXT: ret void ; %1 = shl i64 0, 0 %2 = icmp ult i64 0, 0