diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 5f41bd7d8a61..c51cc2a2fe52 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -4097,7 +4097,7 @@ SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N, if (VT.getScalarType() != MVT::i64) return SDValue(); - // i64 (shl x, C) -> (build_pair 0, (shl x, C -32)) + // i64 (shl x, C) -> (build_pair 0, (shl x, C - 32)) // On some subtargets, 64-bit shift is a quarter rate instruction. In the // common case, splitting this into a move and a 32-bit shift is faster and @@ -4117,12 +4117,12 @@ SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N, ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL, TargetType); } else { - SDValue truncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS); + SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS); const SDValue ShiftMask = DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType); // This AND instruction will clamp out of bounds shift values. // It will also be removed during later instruction selection. - ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, truncShiftAmt, ShiftMask); + ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask); } SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, TargetType, LHS); @@ -4181,50 +4181,105 @@ SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N, SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const { - auto *RHS = dyn_cast(N->getOperand(1)); - if (!RHS) - return SDValue(); - + SDValue RHS = N->getOperand(1); + ConstantSDNode *CRHS = dyn_cast(RHS); EVT VT = N->getValueType(0); SDValue LHS = N->getOperand(0); - unsigned ShiftAmt = RHS->getZExtValue(); SelectionDAG &DAG = DCI.DAG; SDLoc SL(N); + unsigned RHSVal; - // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1) - // this improves the ability to match BFE patterns in isel. - if (LHS.getOpcode() == ISD::AND) { - if (auto *Mask = dyn_cast(LHS.getOperand(1))) { - unsigned MaskIdx, MaskLen; - if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) && - MaskIdx == ShiftAmt) { - return DAG.getNode( - ISD::AND, SL, VT, - DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)), - DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1))); + if (CRHS) { + RHSVal = CRHS->getZExtValue(); + + // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1) + // this improves the ability to match BFE patterns in isel. + if (LHS.getOpcode() == ISD::AND) { + if (auto *Mask = dyn_cast(LHS.getOperand(1))) { + unsigned MaskIdx, MaskLen; + if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) && + MaskIdx == RHSVal) { + return DAG.getNode(ISD::AND, SL, VT, + DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), + N->getOperand(1)), + DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), + N->getOperand(1))); + } } } } - if (VT != MVT::i64) + if (VT.getScalarType() != MVT::i64) return SDValue(); - if (ShiftAmt < 32) + // for C >= 32 + // i64 (srl x, C) -> (build_pair (srl hi_32(x), C -32), 0) + + // On some subtargets, 64-bit shift is a quarter rate instruction. In the + // common case, splitting this into a move and a 32-bit shift is faster and + // the same code size. + KnownBits Known = DAG.computeKnownBits(RHS); + + EVT ElementType = VT.getScalarType(); + EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext()); + EVT TargetType = VT.isVector() ? VT.changeVectorElementType(TargetScalarType) + : TargetScalarType; + + if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits()) return SDValue(); - // srl i64:x, C for C >= 32 - // => - // build_pair (srl hi_32(x), C - 32), 0 - SDValue Zero = DAG.getConstant(0, SL, MVT::i32); + SDValue ShiftAmt; + if (CRHS) { + ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL, + TargetType); + } else { + SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS); + const SDValue ShiftMask = + DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType); + // This AND instruction will clamp out of bounds shift values. + // It will also be removed during later instruction selection. + ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask); + } - SDValue Hi = getHiHalf64(LHS, DAG); + const SDValue Zero = DAG.getConstant(0, SL, TargetScalarType); + EVT ConcatType; + SDValue Hi; + SDLoc LHSSL(LHS); + // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi + if (VT.isVector()) { + unsigned NElts = TargetType.getVectorNumElements(); + ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext()); + SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS); + SmallVector HiOps(NElts); + SmallVector HiAndLoOps; - SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32); - SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); + DAG.ExtractVectorElements(SplitLHS, HiAndLoOps, /*Start=*/0, NElts * 2); + for (unsigned I = 0; I != NElts; ++I) + HiOps[I] = HiAndLoOps[2 * I + 1]; + Hi = DAG.getNode(ISD::BUILD_VECTOR, LHSSL, TargetType, HiOps); + } else { + const SDValue One = DAG.getConstant(1, LHSSL, TargetScalarType); + ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2); + SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS); + Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One); + } - SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero}); + SDValue NewShift = DAG.getNode(ISD::SRL, SL, TargetType, Hi, ShiftAmt); - return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); + SDValue Vec; + if (VT.isVector()) { + unsigned NElts = TargetType.getVectorNumElements(); + SmallVector LoOps; + SmallVector HiAndLoOps(NElts * 2, Zero); + + DAG.ExtractVectorElements(NewShift, LoOps, 0, NElts); + for (unsigned I = 0; I != NElts; ++I) + HiAndLoOps[2 * I] = LoOps[I]; + Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps); + } else { + Vec = DAG.getBuildVector(ConcatType, SL, {NewShift, Zero}); + } + return DAG.getNode(ISD::BITCAST, SL, VT, Vec); } SDValue AMDGPUTargetLowering::performTruncateCombine( @@ -5209,21 +5264,18 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N, break; } - case ISD::SHL: { + case ISD::SHL: + case ISD::SRL: { // Range metadata can be invalidated when loads are converted to legal types // (e.g. v2i64 -> v4i32). - // Try to convert vector shl before type legalization so that range metadata - // can be utilized. + // Try to convert vector shl/srl before type legalization so that range + // metadata can be utilized. if (!(N->getValueType(0).isVector() && DCI.getDAGCombineLevel() == BeforeLegalizeTypes) && DCI.getDAGCombineLevel() < AfterLegalizeDAG) break; - return performShlCombine(N, DCI); - } - case ISD::SRL: { - if (DCI.getDAGCombineLevel() < AfterLegalizeDAG) - break; - + if (N->getOpcode() == ISD::SHL) + return performShlCombine(N, DCI); return performSrlCombine(N, DCI); } case ISD::SRA: { diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll index bb642155cd0a..117f359be0c3 100644 --- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll +++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll @@ -1945,16 +1945,14 @@ define <2 x i64> @lshr_mad_i64_vec(<2 x i64> %arg0) #0 { ; CI-LABEL: lshr_mad_i64_vec: ; CI: ; %bb.0: ; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CI-NEXT: v_mov_b32_e32 v6, v3 -; CI-NEXT: v_mov_b32_e32 v3, v1 -; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: s_mov_b32 s4, 0xffff1c18 -; CI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, s4, v[0:1] -; CI-NEXT: v_mov_b32_e32 v3, v1 +; CI-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, s4, v[0:1] ; CI-NEXT: s_mov_b32 s4, 0xffff1118 -; CI-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s4, v[2:3] +; CI-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, s4, v[2:3] +; CI-NEXT: v_sub_i32_e32 v1, vcc, v5, v1 +; CI-NEXT: v_sub_i32_e32 v3, vcc, v7, v3 ; CI-NEXT: v_mov_b32_e32 v0, v4 -; CI-NEXT: v_mov_b32_e32 v1, v5 +; CI-NEXT: v_mov_b32_e32 v2, v6 ; CI-NEXT: s_setpc_b64 s[30:31] ; ; SI-LABEL: lshr_mad_i64_vec: @@ -1977,44 +1975,28 @@ define <2 x i64> @lshr_mad_i64_vec(<2 x i64> %arg0) #0 { ; GFX9-LABEL: lshr_mad_i64_vec: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v6, v3 -; GFX9-NEXT: v_mov_b32_e32 v3, v1 -; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_mov_b32 s4, 0xffff1c18 -; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, s4, v[0:1] -; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v1, s4, v[0:1] ; GFX9-NEXT: s_mov_b32 s4, 0xffff1118 -; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, s4, v[2:3] +; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v3, s4, v[2:3] +; GFX9-NEXT: v_sub_u32_e32 v1, v5, v1 +; GFX9-NEXT: v_sub_u32_e32 v3, v7, v3 ; GFX9-NEXT: v_mov_b32_e32 v0, v4 -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_mov_b32_e32 v2, v6 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; -; GFX1100-LABEL: lshr_mad_i64_vec: -; GFX1100: ; %bb.0: -; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1100-NEXT: v_mov_b32_e32 v8, v3 -; GFX1100-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v1, 0 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1100-NEXT: v_mad_u64_u32 v[4:5], null, 0xffff1c18, v6, v[0:1] -; GFX1100-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, v4 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1100-NEXT: v_mad_u64_u32 v[6:7], null, 0xffff1118, v8, v[2:3] -; GFX1100-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v2, v6 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1100-NEXT: v_mov_b32_e32 v3, v7 -; GFX1100-NEXT: s_setpc_b64 s[30:31] -; -; GFX1150-LABEL: lshr_mad_i64_vec: -; GFX1150: ; %bb.0: -; GFX1150-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1150-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v5, v1 -; GFX1150-NEXT: v_mov_b32_e32 v1, 0 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1150-NEXT: v_mov_b32_e32 v3, v1 -; GFX1150-NEXT: v_mad_u64_u32 v[0:1], null, 0xffff1c18, v5, v[0:1] -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1150-NEXT: v_mad_u64_u32 v[2:3], null, 0xffff1118, v4, v[2:3] -; GFX1150-NEXT: s_setpc_b64 s[30:31] +; GFX11-LABEL: lshr_mad_i64_vec: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mad_u64_u32 v[4:5], null, 0xffff1c18, v1, v[0:1] +; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, 0xffff1118, v3, v[2:3] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_sub_nc_u32_e32 v1, v5, v1 +; GFX11-NEXT: v_mov_b32_e32 v0, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_sub_nc_u32_e32 v3, v7, v3 +; GFX11-NEXT: v_mov_b32_e32 v2, v6 +; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: lshr_mad_i64_vec: ; GFX12: ; %bb.0: @@ -2023,13 +2005,14 @@ define <2 x i64> @lshr_mad_i64_vec(<2 x i64> %arg0) #0 { ; GFX12-NEXT: s_wait_samplecnt 0x0 ; GFX12-NEXT: s_wait_bvhcnt 0x0 ; GFX12-NEXT: s_wait_kmcnt 0x0 -; GFX12-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v5, v1 -; GFX12-NEXT: v_mov_b32_e32 v1, 0 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-NEXT: v_mov_b32_e32 v3, v1 -; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, 0xffff1c18, v5, v[0:1] -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX12-NEXT: v_mad_co_u64_u32 v[2:3], null, 0xffff1118, v4, v[2:3] +; GFX12-NEXT: v_mad_co_u64_u32 v[4:5], null, 0xffff1c18, v1, v[0:1] +; GFX12-NEXT: v_mad_co_u64_u32 v[6:7], null, 0xffff1118, v3, v[2:3] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-NEXT: v_sub_nc_u32_e32 v1, v5, v1 +; GFX12-NEXT: v_mov_b32_e32 v0, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-NEXT: v_sub_nc_u32_e32 v3, v7, v3 +; GFX12-NEXT: v_mov_b32_e32 v2, v6 ; GFX12-NEXT: s_setpc_b64 s[30:31] %lsh = lshr <2 x i64> %arg0, %mul = mul <2 x i64> %lsh, diff --git a/llvm/test/CodeGen/AMDGPU/srl64_reduce.ll b/llvm/test/CodeGen/AMDGPU/srl64_reduce.ll index 09538c624de7..3567bafe5b1c 100644 --- a/llvm/test/CodeGen/AMDGPU/srl64_reduce.ll +++ b/llvm/test/CodeGen/AMDGPU/srl64_reduce.ll @@ -17,9 +17,9 @@ define i64 @srl_metadata(i64 %arg0, ptr %arg1.ptr) { ; CHECK-LABEL: srl_metadata: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: flat_load_dword v2, v[2:3] +; CHECK-NEXT: flat_load_dword v0, v[2:3] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v2, v[0:1] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v0, v1 ; CHECK-NEXT: v_mov_b32_e32 v1, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %shift.amt = load i64, ptr %arg1.ptr, !range !0, !noundef !{} @@ -30,9 +30,9 @@ define i64 @srl_metadata(i64 %arg0, ptr %arg1.ptr) { define amdgpu_ps i64 @srl_metadata_sgpr_return(i64 inreg %arg0, ptr addrspace(1) inreg %arg1.ptr) { ; CHECK-LABEL: srl_metadata_sgpr_return: ; CHECK: ; %bb.0: -; CHECK-NEXT: s_load_dword s2, s[2:3], 0x0 +; CHECK-NEXT: s_load_dword s0, s[2:3], 0x0 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) -; CHECK-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 +; CHECK-NEXT: s_lshr_b32 s0, s1, s0 ; CHECK-NEXT: s_mov_b32 s1, 0 ; CHECK-NEXT: ; return to shader part epilog %shift.amt = load i64, ptr addrspace(1) %arg1.ptr, !range !0, !noundef !{} @@ -45,9 +45,9 @@ define i64 @srl_exact_metadata(i64 %arg0, ptr %arg1.ptr) { ; CHECK-LABEL: srl_exact_metadata: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: flat_load_dword v2, v[2:3] +; CHECK-NEXT: flat_load_dword v0, v[2:3] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v2, v[0:1] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v0, v1 ; CHECK-NEXT: v_mov_b32_e32 v1, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %shift.amt = load i64, ptr %arg1.ptr, !range !0, !noundef !{} @@ -59,9 +59,9 @@ define i64 @srl_metadata_two_ranges(i64 %arg0, ptr %arg1.ptr) { ; CHECK-LABEL: srl_metadata_two_ranges: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: flat_load_dword v2, v[2:3] +; CHECK-NEXT: flat_load_dword v0, v[2:3] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v2, v[0:1] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v0, v1 ; CHECK-NEXT: v_mov_b32_e32 v1, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %shift.amt = load i64, ptr %arg1.ptr, !range !1, !noundef !{} @@ -106,8 +106,10 @@ define <2 x i64> @srl_v2_metadata(<2 x i64> %arg0, ptr %arg1.ptr) { ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[4:5] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v4, v[0:1] -; CHECK-NEXT: v_lshrrev_b64 v[2:3], v6, v[2:3] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v4, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, v6, v3 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !0, !noundef !{} %srl = lshr <2 x i64> %arg0, %shift.amt @@ -121,8 +123,10 @@ define <2 x i64> @srl_exact_v2_metadata(<2 x i64> %arg0, ptr %arg1.ptr) { ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[4:5] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v4, v[0:1] -; CHECK-NEXT: v_lshrrev_b64 v[2:3], v6, v[2:3] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v4, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, v6, v3 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !0, !noundef !{} %srl = lshr exact <2 x i64> %arg0, %shift.amt @@ -133,12 +137,15 @@ define <3 x i64> @srl_v3_metadata(<3 x i64> %arg0, ptr %arg1.ptr) { ; CHECK-LABEL: srl_v3_metadata: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: flat_load_dword v12, v[6:7] offset:16 +; CHECK-NEXT: flat_load_dword v0, v[6:7] offset:16 ; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[6:7] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_lshrrev_b64 v[4:5], v12, v[4:5] -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v8, v[0:1] -; CHECK-NEXT: v_lshrrev_b64 v[2:3], v10, v[2:3] +; CHECK-NEXT: v_lshrrev_b32_e32 v4, v0, v5 +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v8, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, v10, v3 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 +; CHECK-NEXT: v_mov_b32_e32 v5, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %shift.amt = load <3 x i64>, ptr %arg1.ptr, !range !0, !noundef !{} %srl = lshr <3 x i64> %arg0, %shift.amt @@ -153,11 +160,15 @@ define <4 x i64> @srl_v4_metadata(<4 x i64> %arg0, ptr %arg1.ptr) { ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; CHECK-NEXT: flat_load_dwordx4 v[13:16], v[8:9] offset:16 ; CHECK-NEXT: ; kill: killed $vgpr8 killed $vgpr9 -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v10, v[0:1] -; CHECK-NEXT: v_lshrrev_b64 v[2:3], v12, v[2:3] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v10, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, v12, v3 ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_lshrrev_b64 v[4:5], v13, v[4:5] -; CHECK-NEXT: v_lshrrev_b64 v[6:7], v15, v[6:7] +; CHECK-NEXT: v_lshrrev_b32_e32 v4, v13, v5 +; CHECK-NEXT: v_lshrrev_b32_e32 v6, v15, v7 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 +; CHECK-NEXT: v_mov_b32_e32 v5, 0 +; CHECK-NEXT: v_mov_b32_e32 v7, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %shift.amt = load <4 x i64>, ptr %arg1.ptr, !range !0, !noundef !{} %srl = lshr <4 x i64> %arg0, %shift.amt @@ -337,8 +348,7 @@ define i64 @srl_or32(i64 %arg0, i64 %shift_amt) { ; CHECK-LABEL: srl_or32: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_or_b32_e32 v2, 32, v2 -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v2, v[0:1] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v2, v1 ; CHECK-NEXT: v_mov_b32_e32 v1, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %or = or i64 %shift_amt, 32 @@ -350,10 +360,10 @@ define <2 x i64> @srl_v2_or32(<2 x i64> %arg0, <2 x i64> %shift_amt) { ; CHECK-LABEL: srl_v2_or32: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_or_b32_e32 v5, 32, v6 -; CHECK-NEXT: v_or_b32_e32 v4, 32, v4 -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v4, v[0:1] -; CHECK-NEXT: v_lshrrev_b64 v[2:3], v5, v[2:3] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v4, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, v6, v3 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %or = or <2 x i64> %shift_amt, splat (i64 32) %srl = lshr <2 x i64> %arg0, %or @@ -364,12 +374,12 @@ define <3 x i64> @srl_v3_or32(<3 x i64> %arg0, <3 x i64> %shift_amt) { ; CHECK-LABEL: srl_v3_or32: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_or_b32_e32 v7, 32, v10 -; CHECK-NEXT: v_or_b32_e32 v8, 32, v8 -; CHECK-NEXT: v_or_b32_e32 v6, 32, v6 -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v6, v[0:1] -; CHECK-NEXT: v_lshrrev_b64 v[2:3], v8, v[2:3] -; CHECK-NEXT: v_lshrrev_b64 v[4:5], v7, v[4:5] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v6, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, v8, v3 +; CHECK-NEXT: v_lshrrev_b32_e32 v4, v10, v5 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 +; CHECK-NEXT: v_mov_b32_e32 v5, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %or = or <3 x i64> %shift_amt, splat (i64 32) %srl = lshr <3 x i64> %arg0, %or @@ -380,14 +390,14 @@ define <4 x i64> @srl_v4_or32(<4 x i64> %arg0, <4 x i64> %shift_amt) { ; CHECK-LABEL: srl_v4_or32: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_or_b32_e32 v9, 32, v14 -; CHECK-NEXT: v_or_b32_e32 v11, 32, v12 -; CHECK-NEXT: v_or_b32_e32 v10, 32, v10 -; CHECK-NEXT: v_or_b32_e32 v8, 32, v8 -; CHECK-NEXT: v_lshrrev_b64 v[0:1], v8, v[0:1] -; CHECK-NEXT: v_lshrrev_b64 v[2:3], v10, v[2:3] -; CHECK-NEXT: v_lshrrev_b64 v[4:5], v11, v[4:5] -; CHECK-NEXT: v_lshrrev_b64 v[6:7], v9, v[6:7] +; CHECK-NEXT: v_lshrrev_b32_e32 v0, v8, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, v10, v3 +; CHECK-NEXT: v_lshrrev_b32_e32 v4, v12, v5 +; CHECK-NEXT: v_lshrrev_b32_e32 v6, v14, v7 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 +; CHECK-NEXT: v_mov_b32_e32 v5, 0 +; CHECK-NEXT: v_mov_b32_e32 v7, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %or = or <4 x i64> %shift_amt, splat (i64 32) %srl = lshr <4 x i64> %arg0, %or @@ -400,8 +410,7 @@ define i64 @srl_or32_sgpr(i64 inreg %arg0, i64 inreg %shift_amt) { ; CHECK-LABEL: srl_or32_sgpr: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_or_b32 s4, s18, 32 -; CHECK-NEXT: s_lshr_b64 s[4:5], s[16:17], s4 +; CHECK-NEXT: s_lshr_b32 s4, s17, s18 ; CHECK-NEXT: v_mov_b32_e32 v0, s4 ; CHECK-NEXT: v_mov_b32_e32 v1, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] @@ -413,8 +422,7 @@ define i64 @srl_or32_sgpr(i64 inreg %arg0, i64 inreg %shift_amt) { define amdgpu_ps i64 @srl_or32_sgpr_return(i64 inreg %arg0, i64 inreg %shift_amt) { ; CHECK-LABEL: srl_or32_sgpr_return: ; CHECK: ; %bb.0: -; CHECK-NEXT: s_or_b32 s2, s2, 32 -; CHECK-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 +; CHECK-NEXT: s_lshr_b32 s0, s1, s2 ; CHECK-NEXT: s_mov_b32 s1, 0 ; CHECK-NEXT: ; return to shader part epilog %or = or i64 %shift_amt, 32 @@ -426,14 +434,12 @@ define <2 x i64> @srl_v2_or32_sgpr(<2 x i64> inreg %arg0, <2 x i64> inreg %shift ; CHECK-LABEL: srl_v2_or32_sgpr: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_or_b32 s6, s22, 32 -; CHECK-NEXT: s_or_b32 s4, s20, 32 -; CHECK-NEXT: s_lshr_b64 s[4:5], s[16:17], s4 -; CHECK-NEXT: s_lshr_b64 s[6:7], s[18:19], s6 +; CHECK-NEXT: s_lshr_b32 s4, s17, s20 +; CHECK-NEXT: s_lshr_b32 s5, s19, s22 ; CHECK-NEXT: v_mov_b32_e32 v0, s4 -; CHECK-NEXT: v_mov_b32_e32 v1, s5 -; CHECK-NEXT: v_mov_b32_e32 v2, s6 -; CHECK-NEXT: v_mov_b32_e32 v3, s7 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v2, s5 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %or = or <2 x i64> %shift_amt, splat (i64 32) %srl = lshr <2 x i64> %arg0, %or @@ -444,18 +450,15 @@ define <3 x i64> @srl_v3_or32_sgpr(<3 x i64> inreg %arg0, <3 x i64> inreg %shift ; CHECK-LABEL: srl_v3_or32_sgpr: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_or_b32 s8, s26, 32 -; CHECK-NEXT: s_or_b32 s6, s24, 32 -; CHECK-NEXT: s_or_b32 s4, s22, 32 -; CHECK-NEXT: s_lshr_b64 s[4:5], s[16:17], s4 -; CHECK-NEXT: s_lshr_b64 s[6:7], s[18:19], s6 -; CHECK-NEXT: s_lshr_b64 s[8:9], s[20:21], s8 +; CHECK-NEXT: s_lshr_b32 s4, s17, s22 +; CHECK-NEXT: s_lshr_b32 s5, s19, s24 +; CHECK-NEXT: s_lshr_b32 s6, s21, s26 ; CHECK-NEXT: v_mov_b32_e32 v0, s4 -; CHECK-NEXT: v_mov_b32_e32 v1, s5 -; CHECK-NEXT: v_mov_b32_e32 v2, s6 -; CHECK-NEXT: v_mov_b32_e32 v3, s7 -; CHECK-NEXT: v_mov_b32_e32 v4, s8 -; CHECK-NEXT: v_mov_b32_e32 v5, s9 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v2, s5 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 +; CHECK-NEXT: v_mov_b32_e32 v4, s6 +; CHECK-NEXT: v_mov_b32_e32 v5, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %or = or <3 x i64> %shift_amt, splat (i64 32) %srl = lshr <3 x i64> %arg0, %or @@ -466,20 +469,17 @@ define <4 x i64> @srl_v4_or32_sgpr(<4 x i64> inreg %arg0, <4 x i64> inreg %shift ; CHECK-LABEL: srl_v4_or32_sgpr: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_or_b32_e32 v0, 32, v0 -; CHECK-NEXT: s_or_b32 s8, s28, 32 -; CHECK-NEXT: s_or_b32 s6, s26, 32 -; CHECK-NEXT: s_or_b32 s4, s24, 32 -; CHECK-NEXT: s_lshr_b64 s[4:5], s[16:17], s4 -; CHECK-NEXT: s_lshr_b64 s[6:7], s[18:19], s6 -; CHECK-NEXT: s_lshr_b64 s[8:9], s[20:21], s8 -; CHECK-NEXT: v_lshrrev_b64 v[6:7], v0, s[22:23] +; CHECK-NEXT: s_lshr_b32 s4, s17, s24 +; CHECK-NEXT: s_lshr_b32 s5, s19, s26 +; CHECK-NEXT: s_lshr_b32 s6, s21, s28 +; CHECK-NEXT: v_lshrrev_b32_e64 v6, v0, s23 ; CHECK-NEXT: v_mov_b32_e32 v0, s4 -; CHECK-NEXT: v_mov_b32_e32 v1, s5 -; CHECK-NEXT: v_mov_b32_e32 v2, s6 -; CHECK-NEXT: v_mov_b32_e32 v3, s7 -; CHECK-NEXT: v_mov_b32_e32 v4, s8 -; CHECK-NEXT: v_mov_b32_e32 v5, s9 +; CHECK-NEXT: v_mov_b32_e32 v1, 0 +; CHECK-NEXT: v_mov_b32_e32 v2, s5 +; CHECK-NEXT: v_mov_b32_e32 v3, 0 +; CHECK-NEXT: v_mov_b32_e32 v4, s6 +; CHECK-NEXT: v_mov_b32_e32 v5, 0 +; CHECK-NEXT: v_mov_b32_e32 v7, 0 ; CHECK-NEXT: s_setpc_b64 s[30:31] %or = or <4 x i64> %shift_amt, splat (i64 32) %srl = lshr <4 x i64> %arg0, %or