From 6e1aee4276bbd6f84b34ce449a3c88d2013ca5e4 Mon Sep 17 00:00:00 2001 From: Vigneshwar Jayakumar Date: Wed, 4 Mar 2026 13:31:15 -0600 Subject: [PATCH] [AMDGPU] Select v_bfe_u32 for i8/i16 (and (srl x, c), mask) (#182446) Combine i8 and i16 (and (srl x, c), mask) instructions to v_bfe_32. This optimization is skipped true_i16 targets. resolves issue #179494 --- llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 4 +- .../AMDGPU/AMDGPUInstructionSelector.cpp | 7 +- .../Target/AMDGPU/AMDGPUInstructionSelector.h | 4 +- llvm/lib/Target/AMDGPU/SIInstructions.td | 19 ++- llvm/test/CodeGen/AMDGPU/bfe-i8-i16.ll | 141 ++++++++++++++++++ .../CodeGen/AMDGPU/bitcast_vector_bigint.ll | 9 +- ...ffer-fat-pointers-contents-legalization.ll | 12 +- llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 36 ++--- llvm/test/CodeGen/AMDGPU/permute_i8.ll | 16 +- 9 files changed, 196 insertions(+), 52 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/bfe-i8-i16.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td index cfef04644835..84c0348c1d61 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td @@ -441,8 +441,8 @@ def gi_bitcast_fpimm_to_i32 : GICustomOperandRenderer<"renderBitcastFPImm32">, def gi_bitcast_fpimm_to_i64 : GICustomOperandRenderer<"renderBitcastFPImm64">, GISDNodeXFormEquiv; -def gi_IMMPopCount : GICustomOperandRenderer<"renderPopcntImm">, - GISDNodeXFormEquiv; +def gi_IMMCountTrailingOnes : GICustomOperandRenderer<"renderCountTrailingOnesImm">, + GISDNodeXFormEquiv; def gi_extract_cpol : GICustomOperandRenderer<"renderExtractCPol">, GISDNodeXFormEquiv; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index 61b70dc7585a..81e224355411 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -7224,12 +7224,11 @@ void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB, MIB.addImm(Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue()); } -void AMDGPUInstructionSelector::renderPopcntImm(MachineInstrBuilder &MIB, - const MachineInstr &MI, - int OpIdx) const { +void AMDGPUInstructionSelector::renderCountTrailingOnesImm( + MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT"); - MIB.addImm(MI.getOperand(1).getCImm()->getValue().popcount()); + MIB.addImm(MI.getOperand(1).getCImm()->getValue().countTrailingOnes()); } /// This only really exists to satisfy DAG type checking machinery, so is a diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h index 98c4e7837a1f..cc121632e101 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h @@ -400,8 +400,8 @@ private: renderBitcastFPImm(MIB, MI, OpIdx); } - void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI, - int OpIdx) const; + void renderCountTrailingOnesImm(MachineInstrBuilder &MIB, + const MachineInstr &MI, int OpIdx) const; void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const; void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI, diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 75a0b6b4c34f..bc2dcdf9b591 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -4157,15 +4157,26 @@ def IMMZeroBasedBitfieldMask : ImmLeaf ; -def IMMPopCount : SDNodeXFormgetTargetConstant(llvm::popcount(N->getZExtValue()), SDLoc(N), - MVT::i32); +def IMMZeroBasedBitfieldMask16 : ImmLeaf (Imm) && isMask_32(Imm); +}]>; + +def IMMCountTrailingOnes : SDNodeXFormgetTargetConstant(llvm::countr_one(N->getZExtValue()), + SDLoc(N), MVT::i32); }]>; def : AMDGPUPat < (DivergentBinFrag (i32 (srl i32:$src, i32:$rshift)), IMMZeroBasedBitfieldMask:$mask), - (V_BFE_U32_e64 $src, $rshift, (i32 (IMMPopCount $mask))) + (V_BFE_U32_e64 $src, $rshift, (i32 (IMMCountTrailingOnes $mask))) +>; + +let True16Predicate = NotUseRealTrue16Insts in +def : AMDGPUPat < + (i16 (DivergentBinFrag (srl_oneuse i16:$src, (i16 imm:$rshift)), + IMMZeroBasedBitfieldMask16:$mask)), + (V_BFE_U32_e64 $src, (i32 (as_i32timm $rshift)), (i32 (IMMCountTrailingOnes $mask))) >; // x & ((1 << y) - 1) diff --git a/llvm/test/CodeGen/AMDGPU/bfe-i8-i16.ll b/llvm/test/CodeGen/AMDGPU/bfe-i8-i16.ll new file mode 100644 index 000000000000..d2bf50d125e7 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/bfe-i8-i16.ll @@ -0,0 +1,141 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck %s --check-prefix=GFX9 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck %s --check-prefix=GFX12-TRUE16 + +; Test that isel patterns recognize (and (lshr x, C), mask) for i8/i16 +; and lower it to v_bfe_u32 when real true16 instructions are not used. + +define i16 @bfe_i16(i16 %a) { +; GFX9-LABEL: bfe_i16: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_bfe_u32 v0, v0, 4, 4 +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-TRUE16-LABEL: bfe_i16: +; GFX12-TRUE16: ; %bb.0: +; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-TRUE16-NEXT: v_lshrrev_b16 v0.l, 4, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15 +; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] + %shr = lshr i16 %a, 4 + %and = and i16 %shr, 15 + ret i16 %and +} + +; i8: 4 bits at offset 4. +define i8 @bfe_i8(i8 %a) { +; GFX9-LABEL: bfe_i8: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_bfe_u32 v0, v0, 4, 4 +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-TRUE16-LABEL: bfe_i8: +; GFX12-TRUE16: ; %bb.0: +; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-TRUE16-NEXT: v_lshrrev_b16 v0.l, 4, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15 +; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] + %shr = lshr i8 %a, 4 + %and = and i8 %shr, 15 + ret i8 %and +} + +; Negative: multiple uses of the shifted value should not match the one-use +; narrow BFE pattern. +define i16 @no_bfe_i16_multi_use(i16 %a) { +; GFX9-LABEL: no_bfe_i16_multi_use: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_lshrrev_b16_e32 v0, 4, v0 +; GFX9-NEXT: v_and_b32_e32 v1, 15, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 3, v0 +; GFX9-NEXT: v_xor_b32_e32 v0, v1, v0 +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-TRUE16-LABEL: no_bfe_i16_multi_use: +; GFX12-TRUE16: ; %bb.0: +; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-TRUE16-NEXT: v_lshrrev_b16 v0.l, 4, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_and_b16 v0.h, v0.l, 15 +; GFX12-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3 +; GFX12-TRUE16-NEXT: v_xor_b16 v0.l, v0.h, v0.l +; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] + %shr = lshr i16 %a, 4 + %and0 = and i16 %shr, 15 + %and1 = and i16 %shr, 3 + %xor = xor i16 %and0, %and1 + ret i16 %xor +} + +; Pure uniform (SGPR) case. +define amdgpu_ps i16 @bfe_i16_uniform(i16 inreg %a) { +; GFX9-LABEL: bfe_i16_uniform: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_bfe_u32 s0, s0, 0x40004 +; GFX9-NEXT: ; return to shader part epilog +; +; GFX12-TRUE16-LABEL: bfe_i16_uniform: +; GFX12-TRUE16: ; %bb.0: +; GFX12-TRUE16-NEXT: s_bfe_u32 s0, s0, 0x40004 +; GFX12-TRUE16-NEXT: ; return to shader part epilog + %shr = lshr i16 %a, 4 + %and = and i16 %shr, 15 + ret i16 %and +} + +define amdgpu_ps i8 @bfe_i8_uniform(i8 inreg %a) { +; GFX9-LABEL: bfe_i8_uniform: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_bfe_u32 s0, s0, 0x40004 +; GFX9-NEXT: ; return to shader part epilog +; +; GFX12-TRUE16-LABEL: bfe_i8_uniform: +; GFX12-TRUE16: ; %bb.0: +; GFX12-TRUE16-NEXT: s_bfe_u32 s0, s0, 0x40004 +; GFX12-TRUE16-NEXT: ; return to shader part epilog + %shr = lshr i8 %a, 4 + %and = and i8 %shr, 15 + ret i8 %and +} + +; Vector case: keep the packed shift/and lowering. +define <2 x i16> @bfe_v2i16(<2 x i16> %a) { +; GFX9-LABEL: bfe_v2i16: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_pk_lshrrev_b16 v0, 4, v0 op_sel_hi:[0,1] +; GFX9-NEXT: v_and_b32_e32 v0, 0xf000f, v0 +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-TRUE16-LABEL: bfe_v2i16: +; GFX12-TRUE16: ; %bb.0: +; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-TRUE16-NEXT: v_pk_lshrrev_b16 v0, 4, v0 op_sel_hi:[0,1] +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xf000f, v0 +; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] + %shr = lshr <2 x i16> %a, + %and = and <2 x i16> %shr, + ret <2 x i16> %and +} diff --git a/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll b/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll index f788cd663b30..740d6ba8fe54 100644 --- a/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll +++ b/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll @@ -281,8 +281,7 @@ define <2 x i6> @bitcast_i12_to_v2i6(i12 %int) { ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_and_b32_e32 v2, 63, v0 -; GFX9-NEXT: v_lshrrev_b16_e32 v0, 6, v0 -; GFX9-NEXT: v_and_b32_e32 v1, 63, v0 +; GFX9-NEXT: v_bfe_u32 v1, v0, 6, 6 ; GFX9-NEXT: v_mov_b32_e32 v0, v2 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -306,10 +305,10 @@ define <2 x i6> @bitcast_i12_to_v2i6(i12 %int) { ; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-FAKE16-NEXT: v_lshrrev_b16 v1, 6, v0 -; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 63, v0 +; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 63, v0 +; GFX12-FAKE16-NEXT: v_bfe_u32 v1, v0, 6, 6 ; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 63, v1 +; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v2 ; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31] %bitcast = bitcast i12 %int to <2 x i6> ret <2 x i6> %bitcast diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll index 5967d17c351e..021b15ef09be 100644 --- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll +++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll @@ -3479,16 +3479,14 @@ define <4 x i4> @load_v4i4(ptr addrspace(8) inreg %buf) { ; SDAG: ; %bb.0: ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 0 -; SDAG-NEXT: v_mov_b32_e32 v2, 15 ; SDAG-NEXT: s_waitcnt vmcnt(0) ; SDAG-NEXT: buffer_store_short v0, off, s[0:3], s32 -; SDAG-NEXT: buffer_load_ushort v1, off, s[0:3], s32 +; SDAG-NEXT: buffer_load_ushort v2, off, s[0:3], s32 ; SDAG-NEXT: s_waitcnt vmcnt(0) -; SDAG-NEXT: v_lshrrev_b16_e32 v4, 4, v1 -; SDAG-NEXT: v_and_b32_e32 v0, 15, v1 -; SDAG-NEXT: v_lshrrev_b16_e32 v3, 12, v1 -; SDAG-NEXT: v_and_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD -; SDAG-NEXT: v_and_b32_e32 v1, 15, v4 +; SDAG-NEXT: v_and_b32_e32 v0, 15, v2 +; SDAG-NEXT: v_lshrrev_b16_e32 v3, 12, v2 +; SDAG-NEXT: v_bfe_u32 v1, v2, 4, 4 +; SDAG-NEXT: v_bfe_u32 v2, v2, 8, 4 ; SDAG-NEXT: s_setpc_b64 s[30:31] ; ; GISEL-LABEL: load_v4i4: diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll index 472e79f58969..76eb48539a03 100644 --- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll +++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll @@ -722,14 +722,14 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) { ; SDAG: ; %bb.0: ; %fp-to-i-entry ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SDAG-NEXT: v_mov_b32_e32 v4, v0 -; SDAG-NEXT: v_lshrrev_b16_e32 v8, 7, v4 +; SDAG-NEXT: v_bfe_u32 v8, v4, 7, 8 ; SDAG-NEXT: s_movk_i32 s4, 0x7e ; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: v_mov_b32_e32 v2, 0 ; SDAG-NEXT: v_mov_b32_e32 v1, 0 ; SDAG-NEXT: v_mov_b32_e32 v3, 0 -; SDAG-NEXT: v_cmp_gt_u16_sdwa s[4:5], v8, s4 src0_sel:BYTE_0 src1_sel:DWORD -; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] +; SDAG-NEXT: v_cmp_lt_u16_e32 vcc, s4, v8 +; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc ; SDAG-NEXT: s_cbranch_execz .LBB6_6 ; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size ; SDAG-NEXT: v_ashrrev_i16_e32 v0, 15, v4 @@ -740,15 +740,14 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) { ; SDAG-NEXT: v_ashrrev_i32_e32 v6, 31, v7 ; SDAG-NEXT: v_or_b32_e32 v5, 1, v7 ; SDAG-NEXT: v_or_b32_e32 v4, 0x80, v0 -; SDAG-NEXT: v_cmp_gt_u16_sdwa s[8:9], v8, s5 src0_sel:BYTE_0 src1_sel:DWORD +; SDAG-NEXT: v_cmp_lt_u16_e32 vcc, s5, v8 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 ; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 -; SDAG-NEXT: s_and_saveexec_b64 s[10:11], s[8:9] -; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[10:11] +; SDAG-NEXT: s_and_saveexec_b64 s[8:9], vcc +; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[8:9] ; SDAG-NEXT: s_cbranch_execz .LBB6_3 ; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large -; SDAG-NEXT: s_movk_i32 s5, 0xff7a -; SDAG-NEXT: v_add_u16_sdwa v10, v8, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; SDAG-NEXT: v_add_u16_e32 v10, 0xff7a, v8 ; SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v4 ; SDAG-NEXT: v_mov_b32_e32 v1, s4 ; SDAG-NEXT: v_sub_u32_e32 v2, 64, v10 @@ -788,8 +787,7 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) { ; SDAG-NEXT: .LBB6_3: ; %Flow ; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9] ; SDAG-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small -; SDAG-NEXT: s_movk_i32 s8, 0x86 -; SDAG-NEXT: v_sub_u16_sdwa v0, s8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; SDAG-NEXT: v_sub_u16_e32 v0, 0x86, v8 ; SDAG-NEXT: v_lshrrev_b16_e32 v0, v0, v4 ; SDAG-NEXT: v_mul_hi_i32_i24_e32 v1, v0, v5 ; SDAG-NEXT: v_ashrrev_i32_e32 v2, 31, v1 @@ -936,28 +934,27 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) { ; SDAG: ; %bb.0: ; %fp-to-i-entry ; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SDAG-NEXT: v_mov_b32_e32 v4, v0 -; SDAG-NEXT: v_lshrrev_b16_e32 v5, 7, v4 +; SDAG-NEXT: v_bfe_u32 v5, v4, 7, 8 ; SDAG-NEXT: s_movk_i32 s4, 0x7e ; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: v_mov_b32_e32 v2, 0 ; SDAG-NEXT: v_mov_b32_e32 v1, 0 ; SDAG-NEXT: v_mov_b32_e32 v3, 0 -; SDAG-NEXT: v_cmp_gt_u16_sdwa s[4:5], v5, s4 src0_sel:BYTE_0 src1_sel:DWORD -; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5] +; SDAG-NEXT: v_cmp_lt_u16_e32 vcc, s4, v5 +; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc ; SDAG-NEXT: s_cbranch_execz .LBB7_6 ; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size ; SDAG-NEXT: v_and_b32_e32 v0, 0x7f, v4 ; SDAG-NEXT: s_movk_i32 s4, 0x85 ; SDAG-NEXT: v_or_b32_e32 v4, 0x80, v0 -; SDAG-NEXT: v_cmp_gt_u16_sdwa s[4:5], v5, s4 src0_sel:BYTE_0 src1_sel:DWORD +; SDAG-NEXT: v_cmp_lt_u16_e32 vcc, s4, v5 ; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 ; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 -; SDAG-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] -; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[8:9] +; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc +; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5] ; SDAG-NEXT: s_cbranch_execz .LBB7_3 ; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large -; SDAG-NEXT: s_movk_i32 s4, 0xff7a -; SDAG-NEXT: v_add_u16_sdwa v6, v5, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; SDAG-NEXT: v_add_u16_e32 v6, 0xff7a, v5 ; SDAG-NEXT: s_mov_b32 s4, 0 ; SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v4 ; SDAG-NEXT: v_mov_b32_e32 v1, s4 @@ -979,8 +976,7 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) { ; SDAG-NEXT: .LBB7_3: ; %Flow ; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9] ; SDAG-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small -; SDAG-NEXT: s_movk_i32 s8, 0x86 -; SDAG-NEXT: v_sub_u16_sdwa v0, s8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; SDAG-NEXT: v_sub_u16_e32 v0, 0x86, v5 ; SDAG-NEXT: v_mov_b32_e32 v2, 0 ; SDAG-NEXT: v_lshrrev_b16_e32 v0, v0, v4 ; SDAG-NEXT: v_mov_b32_e32 v3, 0 diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll index f8c8c022567a..d21cda572f5f 100644 --- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll +++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll @@ -2146,12 +2146,12 @@ define hidden void @trunc_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1 ; GFX10-NEXT: global_load_dword v9, v[2:3], off ; GFX10-NEXT: v_mov_b32_e32 v0, 1 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_and_b32_sdwa v1, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD +; GFX10-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_and_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD -; GFX10-NEXT: v_lshlrev_b16 v1, 1, v1 -; GFX10-NEXT: v_lshlrev_b16 v2, 2, v0 -; GFX10-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX10-NEXT: v_bfe_u32 v1, v9, 8, 1 +; GFX10-NEXT: v_lshlrev_b16 v0, 1, v0 +; GFX10-NEXT: v_lshlrev_b16 v2, 2, v1 +; GFX10-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX10-NEXT: v_lshlrev_b16 v1, 3, v4 ; GFX10-NEXT: v_or_b32_e32 v0, v0, v2 ; GFX10-NEXT: v_or_b32_e32 v0, v0, v1 @@ -2175,12 +2175,12 @@ define hidden void @trunc_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1 ; GFX9-NEXT: v_mov_b32_e32 v0, 1 ; GFX9-NEXT: s_mov_b32 s4, 0x50205 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_lshlrev_b16_e32 v3, 3, v4 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_and_b32_sdwa v2, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD ; GFX9-NEXT: v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_bfe_u32 v2, v9, 8, 1 ; GFX9-NEXT: v_lshlrev_b16_e32 v0, 1, v0 ; GFX9-NEXT: v_perm_b32 v1, v9, v4, s4 +; GFX9-NEXT: v_lshlrev_b16_e32 v3, 3, v4 ; GFX9-NEXT: v_lshlrev_b16_e32 v4, 2, v2 ; GFX9-NEXT: v_or_b32_e32 v0, v2, v0 ; GFX9-NEXT: v_or_b32_e32 v0, v0, v4