From 514b38cd7e5ad3969e5d2c7c15a70cd2bf6a6553 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 23 Aug 2023 14:35:28 -0700 Subject: [PATCH] [RISCV] Remove mask size restriction on single source and dual src shuffle costing (try 2) Some callers pass in an empty mask to represent "unknown". We should use the generic costs for these cases. We can add VL=1 costing seperately if desired. Reapplying after revert. A new test had been added, and I'd missed updating it when rebasing before. This is a great happy accident as I hadn't figured out how to get SLP to exercise this case, I'd merely noticed it via inspection. --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 47 ++++++------ .../SLPVectorizer/RISCV/reductions.ll | 71 +++++++++++++------ 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index ecfcf64c7991..99a65265075a 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -324,36 +324,33 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, return LT.first * getLMULCost(LT.second); } } - - // vrgather + cost of generating the mask constant. - // We model this for an unknown mask with a single vrgather. - if (LT.first == 1 && - (LT.second.getScalarSizeInBits() != 8 || - LT.second.getVectorNumElements() <= 256)) { - VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, Tp->getContext()); - InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); - return IndexCost + getVRGatherVVCost(LT.second); - } + } + // vrgather + cost of generating the mask constant. + // We model this for an unknown mask with a single vrgather. + if (LT.second.isFixedLengthVector() && LT.first == 1 && + (LT.second.getScalarSizeInBits() != 8 || + LT.second.getVectorNumElements() <= 256)) { + VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, Tp->getContext()); + InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); + return IndexCost + getVRGatherVVCost(LT.second); } [[fallthrough]]; } case TTI::SK_Transpose: case TTI::SK_PermuteTwoSrc: { - if (Mask.size() >= 2 && LT.second.isFixedLengthVector()) { - // 2 x (vrgather + cost of generating the mask constant) + cost of mask - // register for the second vrgather. We model this for an unknown - // (shuffle) mask. - if (LT.first == 1 && - (LT.second.getScalarSizeInBits() != 8 || - LT.second.getVectorNumElements() <= 256)) { - auto &C = Tp->getContext(); - auto EC = Tp->getElementCount(); - VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C); - VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC); - InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); - InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind); - return 2 * IndexCost + 2 * getVRGatherVVCost(LT.second) + MaskCost; - } + // 2 x (vrgather + cost of generating the mask constant) + cost of mask + // register for the second vrgather. We model this for an unknown + // (shuffle) mask. + if (LT.second.isFixedLengthVector() && LT.first == 1 && + (LT.second.getScalarSizeInBits() != 8 || + LT.second.getVectorNumElements() <= 256)) { + auto &C = Tp->getContext(); + auto EC = Tp->getElementCount(); + VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C); + VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC); + InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind); + InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind); + return 2 * IndexCost + 2 * getVRGatherVVCost(LT.second) + MaskCost; } [[fallthrough]]; } diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll index d6921f6a6eca..35908ee72bc2 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll @@ -1,13 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \ -; RUN: -riscv-v-vector-bits-min=128 -riscv-v-slp-max-vf=0 -S \ -; RUN: | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \ -; RUN: -riscv-v-vector-bits-min=256 -riscv-v-slp-max-vf=0 -S \ -; RUN: | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v \ -; RUN: -riscv-v-vector-bits-min=512 -riscv-v-slp-max-vf=0 -S \ -; RUN: | FileCheck %s --check-prefixes=CHECK +; RUN: -riscv-v-slp-max-vf=0 -S \ +; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL128 +; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v,+zvl256b \ +; RUN: -riscv-v-slp-max-vf=0 -S \ +; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL256 +; RUN: opt < %s -passes=slp-vectorizer -mtriple=riscv64 -mattr=+v,+zvl512b \ +; RUN: -riscv-v-slp-max-vf=0 -S \ +; RUN: | FileCheck %s --check-prefixes=CHECK,ZVL512 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128" target triple = "riscv64" @@ -274,20 +274,47 @@ entry: } define void @reduce_or_2() { -; CHECK-LABEL: @reduce_or_2( -; CHECK-NEXT: [[TMP1:%.*]] = shl i64 0, 0 -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 15 -; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 6 -; CHECK-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]]) -; CHECK-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]] -; CHECK-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]] -; CHECK: 8: -; CHECK-NEXT: ret void -; CHECK: 9: -; CHECK-NEXT: ret void +; ZVL128-LABEL: @reduce_or_2( +; ZVL128-NEXT: [[TMP1:%.*]] = shl i64 0, 0 +; ZVL128-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 15 +; ZVL128-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer +; ZVL128-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 6 +; ZVL128-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer +; ZVL128-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; ZVL128-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]]) +; ZVL128-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]] +; ZVL128-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]] +; ZVL128: 8: +; ZVL128-NEXT: ret void +; ZVL128: 9: +; ZVL128-NEXT: ret void +; +; ZVL256-LABEL: @reduce_or_2( +; ZVL256-NEXT: [[TMP1:%.*]] = shl i64 0, 0 +; ZVL256-NEXT: [[TMP2:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 15 +; ZVL256-NEXT: [[TMP3:%.*]] = icmp ult <16 x i64> [[TMP2]], zeroinitializer +; ZVL256-NEXT: [[TMP4:%.*]] = insertelement <16 x i64> , i64 [[TMP1]], i32 6 +; ZVL256-NEXT: [[TMP5:%.*]] = icmp ult <16 x i64> [[TMP4]], zeroinitializer +; ZVL256-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; ZVL256-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]]) +; ZVL256-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP6]], [[TMP7]] +; ZVL256-NEXT: br i1 [[OP_RDX]], label [[TMP9:%.*]], label [[TMP8:%.*]] +; ZVL256: 8: +; ZVL256-NEXT: ret void +; ZVL256: 9: +; ZVL256-NEXT: ret void +; +; ZVL512-LABEL: @reduce_or_2( +; ZVL512-NEXT: [[TMP1:%.*]] = shl i64 0, 0 +; ZVL512-NEXT: [[TMP2:%.*]] = insertelement <32 x i64> , i64 [[TMP1]], i32 15 +; ZVL512-NEXT: [[TMP3:%.*]] = shufflevector <32 x i64> [[TMP2]], <32 x i64> poison, <32 x i32> +; ZVL512-NEXT: [[TMP4:%.*]] = icmp ult <32 x i64> [[TMP3]], zeroinitializer +; ZVL512-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> [[TMP4]]) +; ZVL512-NEXT: br i1 [[TMP5]], label [[TMP7:%.*]], label [[TMP6:%.*]] +; ZVL512: 6: +; ZVL512-NEXT: ret void +; ZVL512: 7: +; ZVL512-NEXT: ret void ; %1 = shl i64 0, 0 %2 = icmp ult i64 0, 0