shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

PeepholeOpt: Fix losing subregister indexes on full copies (#161310)

Browse Source 
Previously if we had a subregister extract reading from a
full copy, the no-subregister incoming copy would overwrite
the DefSubReg index of the folding context.

There's one ugly rvv regression, but it's a downstream
issue of this; an unnecessary same class reg-to-reg full copy
was avoided.
This commit is contained in:
Matt Arsenault 2025-10-02 13:36:47 +09:00 committed by GitHub
parent ac0e99e191
commit c6e280e7ed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
34 changed files with 2937 additions and 3148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
llvm/lib/CodeGen/PeepholeOptimizer.cpp
View File

@ -1929,7 +1929,27 @@ ValueTrackerResult ValueTracker::getNextSourceFromCopy() {
const MachineOperand &Src = Def->getOperand(1);
if (Src.isUndef())
return ValueTrackerResult();
return ValueTrackerResult(Src.getReg(), Src.getSubReg());
Register SrcReg = Src.getReg();
unsigned SubReg = Src.getSubReg();
if (DefSubReg) {
const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
SubReg = TRI->composeSubRegIndices(SubReg, DefSubReg);
if (SrcReg.isVirtual()) {
// TODO: Try constraining on rewrite if we can
const TargetRegisterClass *RegRC = MRI.getRegClass(SrcReg);
const TargetRegisterClass *SrcWithSubRC =
TRI->getSubClassWithSubReg(RegRC, SubReg);
if (RegRC != SrcWithSubRC)
return ValueTrackerResult();
} else {
if (!TRI->getSubReg(SrcReg, SubReg))
return ValueTrackerResult();
}
}
return ValueTrackerResult(SrcReg, SubReg);
}
ValueTrackerResult ValueTracker::getNextSourceFromBitcast() {

104
llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
View File

@ -1812,26 +1812,26 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v1
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[0:1], v6, s[0:3], null offen
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: buffer_load_b64 v[4:5], v8, s[0:3], null offen
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
; GFX12-NEXT: v_max_num_f64_e32 v[7:8], v[0:1], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
; GFX12-NEXT: v_mov_b32_e32 v0, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1854,27 +1854,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v1
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[0:1], v6, s[0:3], 0 offen
; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b64 v[4:5], v8, s[0:3], 0 offen
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1906,28 +1906,26 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v6, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v8, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: v_mov_b32_e32 v3, v5
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -1937,28 +1935,26 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v6, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v8, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX8-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end

104
llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
View File

@ -1812,26 +1812,26 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v1
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[0:1], v6, s[0:3], null offen
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: buffer_load_b64 v[4:5], v8, s[0:3], null offen
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
; GFX12-NEXT: v_mov_b32_e32 v0, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1854,27 +1854,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v1
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[0:1], v6, s[0:3], 0 offen
; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b64 v[4:5], v8, s[0:3], 0 offen
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen glc
; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1906,28 +1906,26 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v6, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v8, s[16:19], 0 offen
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: v_mov_b32_e32 v3, v5
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -1937,28 +1935,26 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v6, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v8, s[16:19], 0 offen
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end

520
llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
View File

@ -209,48 +209,48 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v5, v2
; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s10, v3, v[1:2]
; GFX8-NEXT: v_mov_b32_e32 v6, s9
; GFX8-NEXT: v_mov_b32_e32 v5, s11
; GFX8-NEXT: v_sub_u32_e32 v6, vcc, s8, v0
; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s11, v4, v[1:2]
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s8, v0
; GFX8-NEXT: v_subb_u32_e64 v6, s[0:1], v6, v1, vcc
; GFX8-NEXT: v_mov_b32_e32 v2, s9
; GFX8-NEXT: v_mov_b32_e32 v5, s11
; GFX8-NEXT: v_subb_u32_e64 v7, s[0:1], v2, v1, vcc
; GFX8-NEXT: v_sub_u32_e64 v0, s[0:1], s9, v1
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v6
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v7
; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v2
; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v6
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v6
; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v7
; GFX8-NEXT: v_subb_u32_e32 v0, vcc, v0, v5, vcc
; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[0:1]
; GFX8-NEXT: v_subrev_u32_e32 v7, vcc, s10, v2
; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v2, s[0:1]
; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s10, v6
; GFX8-NEXT: v_subbrev_u32_e64 v8, s[0:1], 0, v0, vcc
; GFX8-NEXT: v_add_u32_e64 v9, s[0:1], 1, v4
; GFX8-NEXT: v_addc_u32_e64 v10, s[0:1], 0, v3, s[0:1]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v8
; GFX8-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v7
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v2
; GFX8-NEXT: v_subb_u32_e32 v0, vcc, v0, v5, vcc
; GFX8-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v8
; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s10, v7
; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s10, v2
; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[0:1]
; GFX8-NEXT: v_add_u32_e64 v12, s[0:1], 1, v9
; GFX8-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v0, vcc
; GFX8-NEXT: v_addc_u32_e64 v13, s[0:1], 0, v10, s[0:1]
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11
; GFX8-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v9, v10, v13, vcc
; GFX8-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v1
; GFX8-NEXT: v_cndmask_b32_e32 v9, v10, v13, vcc
; GFX8-NEXT: v_cndmask_b32_e64 v0, v4, v0, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v9, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v5, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v14, vcc
; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v4, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
; GFX8-NEXT: v_mov_b32_e32 v4, s4
; GFX8-NEXT: v_cndmask_b32_e64 v1, v3, v9, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v14, vcc
; GFX8-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v2, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
@ -299,7 +299,6 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v0
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v1, vcc
; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s2, v3, 0
; GFX9-NEXT: v_mov_b32_e32 v7, s19
; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s2, v4, v[1:2]
; GFX9-NEXT: v_mul_hi_u32 v6, v3, v0
; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s3, v3, v[1:2]
@ -346,30 +345,30 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: v_add_u32_e32 v3, v4, v3
; GFX9-NEXT: v_add3_u32 v3, v3, v2, v6
; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s18, v3, v[1:2]
; GFX9-NEXT: v_mov_b32_e32 v6, s17
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, s16, v0
; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s19, v5, v[1:2]
; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, s16, v0
; GFX9-NEXT: v_subb_co_u32_e64 v6, s[0:1], v6, v1, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s19, v6
; GFX9-NEXT: v_mov_b32_e32 v2, s17
; GFX9-NEXT: v_mov_b32_e32 v4, s19
; GFX9-NEXT: v_subb_co_u32_e64 v8, s[0:1], v2, v1, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s19, v8
; GFX9-NEXT: v_sub_u32_e32 v0, s17, v1
; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s18, v2
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s19, v6
; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v7, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v8, s[0:1]
; GFX9-NEXT: v_subrev_co_u32_e32 v8, vcc, s18, v2
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s18, v7
; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s19, v8
; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v4, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v2, s[0:1]
; GFX9-NEXT: v_subrev_co_u32_e32 v2, vcc, s18, v7
; GFX9-NEXT: v_subbrev_co_u32_e64 v9, s[0:1], 0, v0, vcc
; GFX9-NEXT: v_add_co_u32_e64 v10, s[0:1], 1, v5
; GFX9-NEXT: v_addc_co_u32_e64 v11, s[0:1], 0, v3, s[0:1]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s19, v9
; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s18, v8
; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v7, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s18, v2
; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v4, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s19, v9
; GFX9-NEXT: v_subrev_co_u32_e32 v7, vcc, s18, v8
; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s18, v2
; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[0:1]
; GFX9-NEXT: v_add_co_u32_e64 v13, s[0:1], 1, v10
; GFX9-NEXT: v_subbrev_co_u32_e32 v15, vcc, 0, v0, vcc
@ -378,14 +377,15 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1)
; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v13, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v10, v11, v14, vcc
; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v1
; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: v_cndmask_b32_e64 v0, v5, v0, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v1, v3, v10, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v7, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v15, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v3, v6, v5, s[0:1]
; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[12:13]
; GFX9-NEXT: global_store_dwordx2 v4, v[2:3], s[14:15]
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v15, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v2, v7, v2, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, v3, s[0:1]
; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[12:13]
; GFX9-NEXT: global_store_dwordx2 v6, v[2:3], s[14:15]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: udivrem_i64:
@ -1070,6 +1070,7 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mul_lo_u32 v3, s8, v1
; GFX8-NEXT: v_mul_hi_u32 v4, s8, v0
; GFX8-NEXT: v_mul_hi_u32 v0, s9, v0
; GFX8-NEXT: v_mov_b32_e32 v5, s13
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4
@ -1082,184 +1083,183 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v3
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v4, v3
; GFX8-NEXT: v_add_u32_e32 v6, vcc, v0, v2
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v0, v2
; GFX8-NEXT: v_mul_hi_u32 v4, s9, v1
; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s12, v6, 0
; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s12, v7, 0
; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v4, v2
; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s12, v7, v[1:2]
; GFX8-NEXT: v_add_u32_e32 v8, vcc, v4, v2
; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s12, v8, v[1:2]
; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s13, v7, v[1:2]
; GFX8-NEXT: v_mov_b32_e32 v3, s9
; GFX8-NEXT: v_sub_u32_e32 v8, vcc, s8, v0
; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s13, v6, v[1:2]
; GFX8-NEXT: v_mov_b32_e32 v4, s13
; GFX8-NEXT: v_subb_u32_e64 v0, s[0:1], v3, v1, vcc
; GFX8-NEXT: v_sub_u32_e64 v1, s[0:1], s9, v1
; GFX8-NEXT: v_sub_u32_e32 v1, vcc, s8, v0
; GFX8-NEXT: v_subb_u32_e64 v0, s[0:1], v3, v2, vcc
; GFX8-NEXT: v_sub_u32_e64 v2, s[0:1], s9, v2
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s13, v0
; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v8
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v1
; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s13, v0
; GFX8-NEXT: v_cndmask_b32_e64 v9, v2, v3, s[0:1]
; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s15
; GFX8-NEXT: v_cvt_f32_u32_e32 v3, s14
; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v1, v4, vcc
; GFX8-NEXT: v_mul_f32_e32 v1, 0x4f800000, v2
; GFX8-NEXT: v_add_f32_e32 v1, v1, v3
; GFX8-NEXT: v_rcp_iflag_f32_e32 v1, v1
; GFX8-NEXT: v_subrev_u32_e32 v10, vcc, s12, v8
; GFX8-NEXT: v_subbrev_u32_e64 v11, s[0:1], 0, v5, vcc
; GFX8-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v1
; GFX8-NEXT: v_mul_f32_e32 v2, 0x2f800000, v1
; GFX8-NEXT: v_trunc_f32_e32 v3, v2
; GFX8-NEXT: v_mul_f32_e32 v2, 0xcf800000, v3
; GFX8-NEXT: v_add_f32_e32 v1, v2, v1
; GFX8-NEXT: v_cvt_u32_f32_e32 v12, v1
; GFX8-NEXT: v_add_u32_e64 v13, s[0:1], 1, v6
; GFX8-NEXT: v_addc_u32_e64 v14, s[0:1], 0, v7, s[0:1]
; GFX8-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s2, v12, 0
; GFX8-NEXT: v_cvt_u32_f32_e32 v15, v3
; GFX8-NEXT: v_cndmask_b32_e64 v9, v3, v4, s[0:1]
; GFX8-NEXT: v_cvt_f32_u32_e32 v3, s15
; GFX8-NEXT: v_cvt_f32_u32_e32 v4, s14
; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v2, v5, vcc
; GFX8-NEXT: v_mul_f32_e32 v2, 0x4f800000, v3
; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
; GFX8-NEXT: v_rcp_iflag_f32_e32 v2, v2
; GFX8-NEXT: v_subrev_u32_e32 v10, vcc, s12, v1
; GFX8-NEXT: v_subbrev_u32_e64 v11, s[0:1], 0, v6, vcc
; GFX8-NEXT: v_mul_f32_e32 v2, 0x5f7ffffc, v2
; GFX8-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
; GFX8-NEXT: v_trunc_f32_e32 v4, v3
; GFX8-NEXT: v_mul_f32_e32 v3, 0xcf800000, v4
; GFX8-NEXT: v_add_f32_e32 v2, v3, v2
; GFX8-NEXT: v_cvt_u32_f32_e32 v12, v2
; GFX8-NEXT: v_add_u32_e64 v13, s[0:1], 1, v7
; GFX8-NEXT: v_addc_u32_e64 v14, s[0:1], 0, v8, s[0:1]
; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v12, 0
; GFX8-NEXT: v_cvt_u32_f32_e32 v15, v4
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s13, v11
; GFX8-NEXT: v_cndmask_b32_e64 v16, 0, -1, s[0:1]
; GFX8-NEXT: v_subb_u32_e32 v4, vcc, v5, v4, vcc
; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v15, v[2:3]
; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v6, v5, vcc
; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s2, v15, v[3:4]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v10
; GFX8-NEXT: v_cndmask_b32_e64 v17, 0, -1, s[0:1]
; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s3, v12, v[2:3]
; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s3, v12, v[3:4]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s13, v11
; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[0:1]
; GFX8-NEXT: v_mul_lo_u32 v3, v15, v1
; GFX8-NEXT: v_mul_lo_u32 v17, v12, v2
; GFX8-NEXT: v_mul_hi_u32 v5, v12, v1
; GFX8-NEXT: v_mul_hi_u32 v1, v15, v1
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v17
; GFX8-NEXT: v_mul_lo_u32 v4, v15, v2
; GFX8-NEXT: v_mul_lo_u32 v17, v12, v3
; GFX8-NEXT: v_mul_hi_u32 v6, v12, v2
; GFX8-NEXT: v_mul_hi_u32 v2, v15, v2
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v17
; GFX8-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v5
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; GFX8-NEXT: v_mul_lo_u32 v5, v15, v2
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v17, v3
; GFX8-NEXT: v_mul_hi_u32 v17, v12, v2
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v5, v1
; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v17
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v6
; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GFX8-NEXT: v_mul_lo_u32 v6, v15, v3
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v17, v4
; GFX8-NEXT: v_mul_hi_u32 v17, v12, v3
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v6, v2
; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v17
; GFX8-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v17
; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v17
; GFX8-NEXT: v_add_u32_e32 v17, vcc, 1, v13
; GFX8-NEXT: v_addc_u32_e32 v18, vcc, 0, v14, vcc
; GFX8-NEXT: v_subrev_u32_e32 v19, vcc, s12, v10
; GFX8-NEXT: v_mul_hi_u32 v2, v15, v2
; GFX8-NEXT: v_subbrev_u32_e32 v20, vcc, 0, v4, vcc
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3
; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v5, v3
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
; GFX8-NEXT: v_add_u32_e32 v12, vcc, v12, v1
; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s2, v12, 0
; GFX8-NEXT: v_addc_u32_e32 v15, vcc, v15, v2, vcc
; GFX8-NEXT: v_mul_hi_u32 v3, v15, v3
; GFX8-NEXT: v_subbrev_u32_e32 v20, vcc, 0, v5, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4
; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v6, v4
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v4
; GFX8-NEXT: v_add_u32_e32 v12, vcc, v12, v2
; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s2, v12, 0
; GFX8-NEXT: v_addc_u32_e32 v15, vcc, v15, v3, vcc
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
; GFX8-NEXT: v_cndmask_b32_e32 v2, v13, v17, vcc
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s2, v15, v[1:2]
; GFX8-NEXT: v_cndmask_b32_e32 v3, v13, v17, vcc
; GFX8-NEXT: v_mov_b32_e32 v2, v5
; GFX8-NEXT: v_mad_u64_u32 v[5:6], s[0:1], s2, v15, v[2:3]
; GFX8-NEXT: v_cndmask_b32_e32 v13, v14, v18, vcc
; GFX8-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v9
; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[2:3], s3, v12, v[4:5]
; GFX8-NEXT: v_cndmask_b32_e64 v1, v6, v2, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v13, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v5, v10, v19, vcc
; GFX8-NEXT: v_mul_lo_u32 v7, v15, v3
; GFX8-NEXT: v_mul_lo_u32 v9, v12, v4
; GFX8-NEXT: v_cndmask_b32_e64 v5, v8, v5, s[0:1]
; GFX8-NEXT: v_mul_hi_u32 v8, v12, v3
; GFX8-NEXT: v_cndmask_b32_e32 v6, v11, v20, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v9
; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v8
; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GFX8-NEXT: v_mul_lo_u32 v8, v15, v4
; GFX8-NEXT: v_mul_hi_u32 v3, v15, v3
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v9, v7
; GFX8-NEXT: v_mad_u64_u32 v[5:6], s[2:3], s3, v12, v[5:6]
; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v3, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v3, v8, v13, s[0:1]
; GFX8-NEXT: v_mul_lo_u32 v7, v15, v4
; GFX8-NEXT: v_mul_lo_u32 v8, v12, v5
; GFX8-NEXT: v_mul_hi_u32 v9, v12, v4
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v8, v3
; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v19, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v10, v11, v20, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v8
; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v9
; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v9
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v9
; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GFX8-NEXT: v_mul_lo_u32 v9, v15, v5
; GFX8-NEXT: v_mul_hi_u32 v4, v15, v4
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v7
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v8, v7
; GFX8-NEXT: v_mul_hi_u32 v8, v12, v5
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v9, v4
; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v8
; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v8, vcc, v9, v8
; GFX8-NEXT: v_mul_hi_u32 v5, v15, v5
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v7
; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v8, v7
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v7
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v12, v3
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, v15, v4, vcc
; GFX8-NEXT: v_mul_lo_u32 v7, s11, v3
; GFX8-NEXT: v_mul_lo_u32 v8, s10, v4
; GFX8-NEXT: v_cndmask_b32_e64 v6, v0, v6, s[0:1]
; GFX8-NEXT: v_mul_hi_u32 v0, s10, v3
; GFX8-NEXT: v_mul_hi_u32 v3, s11, v3
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v8
; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v7, v0
; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v7
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v12, v4
; GFX8-NEXT: v_addc_u32_e32 v5, vcc, v15, v5, vcc
; GFX8-NEXT: v_mul_lo_u32 v7, s11, v4
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v8, v0
; GFX8-NEXT: v_mul_hi_u32 v8, s10, v4
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v7, v3
; GFX8-NEXT: v_mul_lo_u32 v8, s10, v5
; GFX8-NEXT: v_cndmask_b32_e64 v6, v1, v6, s[0:1]
; GFX8-NEXT: v_mul_hi_u32 v1, s10, v4
; GFX8-NEXT: v_mul_hi_u32 v4, s11, v4
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v8
; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v7, v1
; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; GFX8-NEXT: v_mul_lo_u32 v7, s11, v5
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v8, v1
; GFX8-NEXT: v_mul_hi_u32 v8, s10, v5
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v7, v4
; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v8
; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v8
; GFX8-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v8
; GFX8-NEXT: v_add_u32_e32 v9, vcc, v3, v0
; GFX8-NEXT: v_mul_hi_u32 v8, s11, v4
; GFX8-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s14, v9, 0
; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v7, v0
; GFX8-NEXT: v_add_u32_e32 v10, vcc, v8, v0
; GFX8-NEXT: v_mov_b32_e32 v0, v4
; GFX8-NEXT: v_mad_u64_u32 v[7:8], s[0:1], s14, v10, v[0:1]
; GFX8-NEXT: v_mov_b32_e32 v4, s11
; GFX8-NEXT: v_mov_b32_e32 v0, s15
; GFX8-NEXT: v_mad_u64_u32 v[7:8], s[0:1], s15, v9, v[7:8]
; GFX8-NEXT: v_sub_u32_e32 v8, vcc, s10, v3
; GFX8-NEXT: v_subb_u32_e64 v11, s[0:1], v4, v7, vcc
; GFX8-NEXT: v_sub_u32_e64 v3, s[0:1], s11, v7
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v11
; GFX8-NEXT: v_add_u32_e32 v11, vcc, v4, v1
; GFX8-NEXT: v_mul_hi_u32 v8, s11, v5
; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[2:3], s14, v11, 0
; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v7, v1
; GFX8-NEXT: v_add_u32_e32 v12, vcc, v8, v1
; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: v_mad_u64_u32 v[8:9], s[2:3], s14, v12, v[1:2]
; GFX8-NEXT: v_cndmask_b32_e64 v7, v0, v10, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v5, s15
; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s15, v11, v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v1, s11
; GFX8-NEXT: v_sub_u32_e32 v8, vcc, s10, v4
; GFX8-NEXT: v_subb_u32_e64 v1, s[0:1], v1, v0, vcc
; GFX8-NEXT: v_sub_u32_e64 v0, s[0:1], s11, v0
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v1
; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s14, v8
; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v11
; GFX8-NEXT: v_subb_u32_e32 v3, vcc, v3, v0, vcc
; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v7, s[0:1]
; GFX8-NEXT: v_subrev_u32_e32 v7, vcc, s14, v8
; GFX8-NEXT: v_subbrev_u32_e64 v12, s[0:1], 0, v3, vcc
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v12
; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v1
; GFX8-NEXT: v_subb_u32_e32 v0, vcc, v0, v5, vcc
; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v9, s[0:1]
; GFX8-NEXT: v_subrev_u32_e32 v9, vcc, s14, v8
; GFX8-NEXT: v_subbrev_u32_e64 v10, s[0:1], 0, v0, vcc
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v10
; GFX8-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s14, v7
; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s14, v9
; GFX8-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v12
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v10
; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[0:1]
; GFX8-NEXT: v_add_u32_e64 v14, s[0:1], 1, v9
; GFX8-NEXT: v_subb_u32_e32 v0, vcc, v3, v0, vcc
; GFX8-NEXT: v_addc_u32_e64 v15, s[0:1], 0, v10, s[0:1]
; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v14
; GFX8-NEXT: v_add_u32_e64 v14, s[0:1], 1, v11
; GFX8-NEXT: v_subb_u32_e32 v0, vcc, v0, v5, vcc
; GFX8-NEXT: v_addc_u32_e64 v15, s[0:1], 0, v12, s[0:1]
; GFX8-NEXT: v_add_u32_e32 v5, vcc, 1, v14
; GFX8-NEXT: v_addc_u32_e32 v16, vcc, 0, v15, vcc
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13
; GFX8-NEXT: v_subrev_u32_e64 v13, s[0:1], s14, v7
; GFX8-NEXT: v_subrev_u32_e64 v13, s[0:1], s14, v9
; GFX8-NEXT: v_subbrev_u32_e64 v0, s[0:1], 0, v0, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v3, v14, v3, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v14, v15, v16, vcc
; GFX8-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4
; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v3, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v4, v10, v14, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v10, s5
; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v13, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc
; GFX8-NEXT: v_mov_b32_e32 v9, s4
; GFX8-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v8, v11, v0, s[0:1]
; GFX8-NEXT: flat_store_dwordx4 v[9:10], v[1:4]
; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v13, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v5, v14, v5, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v14, v15, v16, vcc
; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v9, v1, v0, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_cndmask_b32_e64 v4, v11, v5, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e64 v5, v12, v14, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[5:8]
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[6:9]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udivrem_v2i64:
@ -1355,11 +1355,11 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_add_u32_e32 v3, v4, v3
; GFX9-NEXT: v_add3_u32 v8, v3, v2, v5
; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[0:1], s4, v8, v[1:2]
; GFX9-NEXT: v_mov_b32_e32 v4, s17
; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s5, v7, v[1:2]
; GFX9-NEXT: v_mov_b32_e32 v3, s17
; GFX9-NEXT: v_sub_co_u32_e32 v1, vcc, s16, v0
; GFX9-NEXT: v_subb_co_u32_e64 v0, s[0:1], v4, v2, vcc
; GFX9-NEXT: v_subb_co_u32_e64 v0, s[0:1], v3, v2, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s5, v0
; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s4, v1
@ -1387,7 +1387,7 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_cvt_u32_f32_e32 v15, v4
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s5, v11
; GFX9-NEXT: v_cndmask_b32_e64 v16, 0, -1, s[0:1]
; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v6, v5, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v6, v5, vcc
; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[0:1], s2, v15, v[3:4]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s4, v10
; GFX9-NEXT: v_cndmask_b32_e64 v17, 0, -1, s[0:1]
@ -1396,128 +1396,128 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[0:1]
; GFX9-NEXT: v_mul_lo_u32 v4, v15, v2
; GFX9-NEXT: v_mul_lo_u32 v17, v12, v3
; GFX9-NEXT: v_mul_hi_u32 v6, v12, v2
; GFX9-NEXT: v_mul_hi_u32 v5, v12, v2
; GFX9-NEXT: v_mul_hi_u32 v2, v15, v2
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v17
; GFX9-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v6
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v5
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GFX9-NEXT: v_mul_lo_u32 v6, v15, v3
; GFX9-NEXT: v_mul_lo_u32 v5, v15, v3
; GFX9-NEXT: v_add_u32_e32 v4, v17, v4
; GFX9-NEXT: v_mul_hi_u32 v17, v12, v3
; GFX9-NEXT: v_mul_hi_u32 v3, v15, v3
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v6, v2
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v5, v2
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v17
; GFX9-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
; GFX9-NEXT: v_add_u32_e32 v6, v6, v17
; GFX9-NEXT: v_add_u32_e32 v5, v5, v17
; GFX9-NEXT: v_add_co_u32_e32 v17, vcc, 1, v13
; GFX9-NEXT: v_addc_co_u32_e32 v18, vcc, 0, v14, vcc
; GFX9-NEXT: v_subrev_co_u32_e32 v19, vcc, s4, v10
; GFX9-NEXT: v_subbrev_co_u32_e32 v20, vcc, 0, v5, vcc
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v12, v2
; GFX9-NEXT: v_add3_u32 v3, v6, v4, v3
; GFX9-NEXT: v_add3_u32 v3, v5, v4, v3
; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[0:1], s2, v12, 0
; GFX9-NEXT: v_addc_co_u32_e32 v15, vcc, v15, v3, vcc
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
; GFX9-NEXT: v_cndmask_b32_e32 v3, v13, v17, vcc
; GFX9-NEXT: v_mov_b32_e32 v2, v5
; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[0:1], s2, v15, v[2:3]
; GFX9-NEXT: v_cndmask_b32_e32 v13, v14, v18, vcc
; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[0:1], s2, v15, v[2:3]
; GFX9-NEXT: v_subrev_co_u32_e32 v19, vcc, s4, v10
; GFX9-NEXT: v_subbrev_co_u32_e32 v20, vcc, 0, v6, vcc
; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[0:1], s3, v12, v[2:3]
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
; GFX9-NEXT: v_cndmask_b32_e32 v13, v13, v17, vcc
; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v9
; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[2:3], s3, v12, v[5:6]
; GFX9-NEXT: v_cndmask_b32_e64 v2, v7, v3, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, v13, s[0:1]
; GFX9-NEXT: v_mul_lo_u32 v7, v15, v4
; GFX9-NEXT: v_mul_lo_u32 v8, v12, v5
; GFX9-NEXT: v_cndmask_b32_e32 v6, v10, v19, vcc
; GFX9-NEXT: v_mul_hi_u32 v10, v12, v4
; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v20, vcc
; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v8
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v10
; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GFX9-NEXT: v_mul_lo_u32 v10, v15, v5
; GFX9-NEXT: v_cndmask_b32_e64 v2, v7, v13, s[0:1]
; GFX9-NEXT: v_mul_lo_u32 v6, v15, v4
; GFX9-NEXT: v_mul_lo_u32 v7, v12, v5
; GFX9-NEXT: v_mul_hi_u32 v9, v12, v4
; GFX9-NEXT: v_mul_hi_u32 v4, v15, v4
; GFX9-NEXT: v_add_u32_e32 v7, v8, v7
; GFX9-NEXT: v_mul_hi_u32 v8, v12, v5
; GFX9-NEXT: v_cndmask_b32_e32 v14, v14, v18, vcc
; GFX9-NEXT: v_add_co_u32_e64 v6, s[2:3], v6, v7
; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[2:3]
; GFX9-NEXT: v_add_co_u32_e64 v6, s[2:3], v6, v9
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[2:3]
; GFX9-NEXT: v_mul_lo_u32 v9, v15, v5
; GFX9-NEXT: v_add_u32_e32 v6, v7, v6
; GFX9-NEXT: v_mul_hi_u32 v7, v12, v5
; GFX9-NEXT: v_mul_hi_u32 v5, v15, v5
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v10, v4
; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v7
; GFX9-NEXT: v_add_u32_e32 v8, v10, v8
; GFX9-NEXT: v_add_co_u32_e64 v4, s[2:3], v9, v4
; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[2:3]
; GFX9-NEXT: v_add_co_u32_e64 v4, s[2:3], v4, v7
; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[2:3]
; GFX9-NEXT: v_add_co_u32_e64 v4, s[2:3], v4, v6
; GFX9-NEXT: v_add_u32_e32 v7, v9, v7
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[2:3]
; GFX9-NEXT: v_add3_u32 v5, v7, v6, v5
; GFX9-NEXT: v_add_co_u32_e64 v4, s[2:3], v12, v4
; GFX9-NEXT: v_addc_co_u32_e64 v5, s[2:3], v15, v5, s[2:3]
; GFX9-NEXT: v_mul_lo_u32 v6, s19, v4
; GFX9-NEXT: v_mul_lo_u32 v7, s18, v5
; GFX9-NEXT: v_mul_hi_u32 v9, s18, v4
; GFX9-NEXT: v_cndmask_b32_e64 v3, v8, v14, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v19, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v10, v11, v20, vcc
; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v7
; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GFX9-NEXT: v_add3_u32 v5, v8, v7, v5
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v12, v4
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v15, v5, vcc
; GFX9-NEXT: v_mul_lo_u32 v7, s19, v4
; GFX9-NEXT: v_mul_lo_u32 v8, s18, v5
; GFX9-NEXT: v_cndmask_b32_e64 v6, v1, v6, s[0:1]
; GFX9-NEXT: v_mul_hi_u32 v1, s18, v4
; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v6, v9
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GFX9-NEXT: v_mul_lo_u32 v9, s19, v5
; GFX9-NEXT: v_mul_hi_u32 v4, s19, v4
; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v8
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v1, vcc, v7, v1
; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; GFX9-NEXT: v_mul_lo_u32 v7, s19, v5
; GFX9-NEXT: v_add_u32_e32 v1, v8, v1
; GFX9-NEXT: v_mul_hi_u32 v8, s18, v5
; GFX9-NEXT: v_mul_hi_u32 v12, s19, v5
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v7, v4
; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v8
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v4, v1
; GFX9-NEXT: v_add_u32_e32 v6, v7, v6
; GFX9-NEXT: v_mul_hi_u32 v7, s18, v5
; GFX9-NEXT: v_mul_hi_u32 v13, s19, v5
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v9, v4
; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v7
; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v4, v6
; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[2:3], s6, v11, 0
; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v7, v0, v9, s[0:1]
; GFX9-NEXT: v_add_u32_e32 v0, v10, v8
; GFX9-NEXT: v_add3_u32 v8, v0, v1, v12
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s6, v8, v[0:1]
; GFX9-NEXT: v_mov_b32_e32 v9, s19
; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v6, v1, v8, s[0:1]
; GFX9-NEXT: v_add_u32_e32 v1, v9, v7
; GFX9-NEXT: v_add3_u32 v12, v1, v12, v13
; GFX9-NEXT: v_mov_b32_e32 v1, v5
; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[2:3], s6, v12, v[1:2]
; GFX9-NEXT: v_cndmask_b32_e64 v7, v0, v10, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v5, s7
; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s7, v11, v[0:1]
; GFX9-NEXT: v_sub_co_u32_e32 v1, vcc, s18, v4
; GFX9-NEXT: v_subb_co_u32_e64 v9, s[0:1], v9, v0, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v9
; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[0:1], s7, v11, v[8:9]
; GFX9-NEXT: v_mov_b32_e32 v1, s19
; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, s18, v4
; GFX9-NEXT: v_subb_co_u32_e64 v1, s[0:1], v1, v0, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v1
; GFX9-NEXT: v_sub_u32_e32 v0, s19, v0
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v1
; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v9
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v8
; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v1
; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v5, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v10, s[0:1]
; GFX9-NEXT: v_subrev_co_u32_e32 v10, vcc, s6, v1
; GFX9-NEXT: v_subbrev_co_u32_e64 v12, s[0:1], 0, v0, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v12
; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v9, s[0:1]
; GFX9-NEXT: v_subrev_co_u32_e32 v9, vcc, s6, v8
; GFX9-NEXT: v_subbrev_co_u32_e64 v10, s[0:1], 0, v0, vcc
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v10
; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v10
; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s6, v9
; GFX9-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v12
; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s7, v10
; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[0:1]
; GFX9-NEXT: v_add_co_u32_e64 v14, s[0:1], 1, v11
; GFX9-NEXT: v_subb_co_u32_e32 v0, vcc, v0, v5, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v15, s[0:1], 0, v8, s[0:1]
; GFX9-NEXT: v_addc_co_u32_e64 v15, s[0:1], 0, v12, s[0:1]
; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, 1, v14
; GFX9-NEXT: v_addc_co_u32_e32 v16, vcc, 0, v15, vcc
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13
; GFX9-NEXT: v_cndmask_b32_e32 v5, v14, v5, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v14, v15, v16, vcc
; GFX9-NEXT: v_subrev_co_u32_e64 v15, s[0:1], s6, v10
; GFX9-NEXT: v_subrev_co_u32_e64 v15, s[0:1], s6, v9
; GFX9-NEXT: v_subbrev_co_u32_e64 v0, s[0:1], 0, v0, s[0:1]
; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v4
; GFX9-NEXT: v_mov_b32_e32 v13, 0
; GFX9-NEXT: v_cndmask_b32_e64 v4, v11, v5, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v5, v8, v14, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e32 v8, v10, v15, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v8, v1, v8, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v9, v9, v0, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v5, v12, v14, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v15, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v9, v1, v0, s[0:1]
; GFX9-NEXT: global_store_dwordx4 v13, v[2:5], s[12:13]
; GFX9-NEXT: global_store_dwordx4 v13, v[6:9], s[14:15]
; GFX9-NEXT: s_endpgm

1
llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
View File

@ -1483,7 +1483,6 @@ define void @flat_atomic_xchg_i64_noret_av(ptr %ptr) #0 {
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB20_2
; GFX90A-NEXT: .LBB20_4: ; %atomicrmw.private

1659
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
View File

File diff suppressed because it is too large Load Diff

133
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
View File

@ -85072,13 +85072,13 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
@ -85086,20 +85086,20 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
@ -85119,18 +85119,18 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 24, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v15
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 24, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v7
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v7
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v5
@ -85159,19 +85159,19 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v7.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v9.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v10.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v11.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v12.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.h, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v13.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.h, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.h, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v14.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.h, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.h, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v15.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.h, v16.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v16.h
@ -85345,29 +85345,29 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v68.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v67.h
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v67.h
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[23:24]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[49:50], 24, v[21:22]
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v2, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[49:50], 24, v[21:22]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[19:20]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[51:52], 24, v[17:18]
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v83, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v82, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v8 :: v_dual_lshlrev_b32 v5, 16, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v83.h
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 24, v24
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v82, v1, v2 :: v_dual_lshlrev_b32 v1, 16, v11
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v82.h
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[51:52], 24, v[17:18]
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v80, v1, v2 :: v_dual_lshlrev_b32 v1, 16, v11
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
@ -85384,82 +85384,81 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v23
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v80.h
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 24, v24
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v27, v2, v3 :: v_dual_add_f32 v2, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v82.h
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v28
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v97, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v97.h
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v9
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v15
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v96, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v9
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v97.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v87, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v28
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[27:28]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[25:26]
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v96.h
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v28
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v87.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v3, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v33
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[25:26]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v33
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v28
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v27
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v23
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v112, v4, v6 :: v_dual_add_f32 v1, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v8 :: v_dual_lshlrev_b32 v5, 16, v15
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v112.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v103, v2, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v15
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v10, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v27
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v32
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v114, v7, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v103, v2, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v10, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v113, v7, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v4, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v103.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v37, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v114.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v113.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 24, v38
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v38
@ -85524,7 +85523,7 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.l, v7.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v68.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v113.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v114.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v98.l
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v27.h
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v31, v6
@ -85541,12 +85540,12 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v8.l, v8.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v9.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v83.h
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v82.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v100.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v30.l
; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v33.h
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v31, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v80.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v9.l, v9.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v10.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v31.h
@ -85560,14 +85559,14 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v11.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v97.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v87.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v96.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v69.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v31, v10
; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v11.l, v11.h
; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v12.l, v12.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v82.h
; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v80.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v86.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v31, v11
@ -85581,14 +85580,14 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v13.l, v13.h
; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.l, v14.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v96.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v81.l
; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v87.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v83.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v31, v13
; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v14.l, v14.h
; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.l, v15.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v31.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v114.h
; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v113.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v71.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v31, v14

96
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
View File

@ -11261,8 +11261,8 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s5, s17, 24
; VI-NEXT: s_lshr_b32 s8, s17, 16
; VI-NEXT: s_lshr_b32 s9, s17, 8
; VI-NEXT: s_lshr_b32 s10, s16, 16
; VI-NEXT: s_lshr_b32 s10, s17, 8
; VI-NEXT: s_lshr_b32 s9, s16, 16
; VI-NEXT: s_lshr_b32 s11, s16, 8
; VI-NEXT: s_cbranch_execnz .LBB85_4
; VI-NEXT: .LBB85_2: ; %cmp.true
@ -11277,9 +11277,9 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; VI-NEXT: s_branch .LBB85_5
; VI-NEXT: .LBB85_3:
; VI-NEXT: ; implicit-def: $sgpr11
; VI-NEXT: ; implicit-def: $sgpr10
; VI-NEXT: ; implicit-def: $sgpr4
; VI-NEXT: ; implicit-def: $sgpr9
; VI-NEXT: ; implicit-def: $sgpr4
; VI-NEXT: ; implicit-def: $sgpr10
; VI-NEXT: ; implicit-def: $sgpr8
; VI-NEXT: ; implicit-def: $sgpr5
; VI-NEXT: s_branch .LBB85_2
@ -11287,8 +11287,8 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; VI-NEXT: v_mov_b32_e32 v8, s16
; VI-NEXT: v_mov_b32_e32 v9, s17
; VI-NEXT: v_mov_b32_e32 v1, s11
; VI-NEXT: v_mov_b32_e32 v2, s10
; VI-NEXT: v_mov_b32_e32 v5, s9
; VI-NEXT: v_mov_b32_e32 v2, s9
; VI-NEXT: v_mov_b32_e32 v5, s10
; VI-NEXT: v_mov_b32_e32 v6, s8
; VI-NEXT: v_mov_b32_e32 v7, s5
; VI-NEXT: v_mov_b32_e32 v3, s4
@ -11306,8 +11306,8 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s5, s17, 24
; GFX9-NEXT: s_lshr_b32 s8, s17, 16
; GFX9-NEXT: s_lshr_b32 s9, s17, 8
; GFX9-NEXT: s_lshr_b32 s10, s16, 16
; GFX9-NEXT: s_lshr_b32 s10, s17, 8
; GFX9-NEXT: s_lshr_b32 s9, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
; GFX9-NEXT: s_cbranch_execnz .LBB85_4
; GFX9-NEXT: .LBB85_2: ; %cmp.true
@ -11322,9 +11322,9 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX9-NEXT: s_branch .LBB85_5
; GFX9-NEXT: .LBB85_3:
; GFX9-NEXT: ; implicit-def: $sgpr11
; GFX9-NEXT: ; implicit-def: $sgpr10
; GFX9-NEXT: ; implicit-def: $sgpr4
; GFX9-NEXT: ; implicit-def: $sgpr9
; GFX9-NEXT: ; implicit-def: $sgpr4
; GFX9-NEXT: ; implicit-def: $sgpr10
; GFX9-NEXT: ; implicit-def: $sgpr8
; GFX9-NEXT: ; implicit-def: $sgpr5
; GFX9-NEXT: s_branch .LBB85_2
@ -11332,8 +11332,8 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX9-NEXT: v_mov_b32_e32 v8, s16
; GFX9-NEXT: v_mov_b32_e32 v9, s17
; GFX9-NEXT: v_mov_b32_e32 v1, s11
; GFX9-NEXT: v_mov_b32_e32 v2, s10
; GFX9-NEXT: v_mov_b32_e32 v5, s9
; GFX9-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NEXT: v_mov_b32_e32 v5, s10
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: v_mov_b32_e32 v3, s4
@ -11352,8 +11352,8 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 24
; GFX11-NEXT: s_lshr_b32 s5, s1, 16
; GFX11-NEXT: s_lshr_b32 s6, s1, 8
; GFX11-NEXT: s_lshr_b32 s7, s0, 16
; GFX11-NEXT: s_lshr_b32 s7, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s8, s0, 8
; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_vccnz .LBB85_4
@ -11370,16 +11370,16 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX11-NEXT: s_branch .LBB85_5
; GFX11-NEXT: .LBB85_3:
; GFX11-NEXT: ; implicit-def: $sgpr8
; GFX11-NEXT: ; implicit-def: $sgpr7
; GFX11-NEXT: ; implicit-def: $sgpr2
; GFX11-NEXT: ; implicit-def: $sgpr6
; GFX11-NEXT: ; implicit-def: $sgpr2
; GFX11-NEXT: ; implicit-def: $sgpr7
; GFX11-NEXT: ; implicit-def: $sgpr5
; GFX11-NEXT: ; implicit-def: $sgpr3
; GFX11-NEXT: s_branch .LBB85_2
; GFX11-NEXT: .LBB85_4:
; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s7
; GFX11-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_mov_b32_e32 v7, s3
; GFX11-NEXT: v_mov_b32_e32 v3, s2
; GFX11-NEXT: .LBB85_5: ; %end
@ -13517,8 +13517,8 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s5, s17, 24
; GFX9-NEXT: s_lshr_b32 s8, s17, 16
; GFX9-NEXT: s_lshr_b32 s9, s17, 8
; GFX9-NEXT: s_lshr_b32 s10, s16, 16
; GFX9-NEXT: s_lshr_b32 s10, s17, 8
; GFX9-NEXT: s_lshr_b32 s9, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
; GFX9-NEXT: s_cbranch_execnz .LBB97_4
; GFX9-NEXT: .LBB97_2: ; %cmp.true
@ -13533,9 +13533,9 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX9-NEXT: s_branch .LBB97_5
; GFX9-NEXT: .LBB97_3:
; GFX9-NEXT: ; implicit-def: $sgpr11
; GFX9-NEXT: ; implicit-def: $sgpr10
; GFX9-NEXT: ; implicit-def: $sgpr4
; GFX9-NEXT: ; implicit-def: $sgpr9
; GFX9-NEXT: ; implicit-def: $sgpr4
; GFX9-NEXT: ; implicit-def: $sgpr10
; GFX9-NEXT: ; implicit-def: $sgpr8
; GFX9-NEXT: ; implicit-def: $sgpr5
; GFX9-NEXT: s_branch .LBB97_2
@ -13543,8 +13543,8 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX9-NEXT: v_mov_b32_e32 v8, s16
; GFX9-NEXT: v_mov_b32_e32 v9, s17
; GFX9-NEXT: v_mov_b32_e32 v1, s11
; GFX9-NEXT: v_mov_b32_e32 v2, s10
; GFX9-NEXT: v_mov_b32_e32 v5, s9
; GFX9-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NEXT: v_mov_b32_e32 v5, s10
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: v_mov_b32_e32 v3, s4
@ -13563,8 +13563,8 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 24
; GFX11-NEXT: s_lshr_b32 s5, s1, 16
; GFX11-NEXT: s_lshr_b32 s6, s1, 8
; GFX11-NEXT: s_lshr_b32 s7, s0, 16
; GFX11-NEXT: s_lshr_b32 s7, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s8, s0, 8
; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_vccnz .LBB97_4
@ -13581,16 +13581,16 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX11-NEXT: s_branch .LBB97_5
; GFX11-NEXT: .LBB97_3:
; GFX11-NEXT: ; implicit-def: $sgpr8
; GFX11-NEXT: ; implicit-def: $sgpr7
; GFX11-NEXT: ; implicit-def: $sgpr2
; GFX11-NEXT: ; implicit-def: $sgpr6
; GFX11-NEXT: ; implicit-def: $sgpr2
; GFX11-NEXT: ; implicit-def: $sgpr7
; GFX11-NEXT: ; implicit-def: $sgpr5
; GFX11-NEXT: ; implicit-def: $sgpr3
; GFX11-NEXT: s_branch .LBB97_2
; GFX11-NEXT: .LBB97_4:
; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s7
; GFX11-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_mov_b32_e32 v7, s3
; GFX11-NEXT: v_mov_b32_e32 v3, s2
; GFX11-NEXT: .LBB97_5: ; %end
@ -15345,8 +15345,8 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX9-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; GFX9-NEXT: s_lshr_b32 s5, s17, 24
; GFX9-NEXT: s_lshr_b32 s8, s17, 16
; GFX9-NEXT: s_lshr_b32 s9, s17, 8
; GFX9-NEXT: s_lshr_b32 s10, s16, 16
; GFX9-NEXT: s_lshr_b32 s10, s17, 8
; GFX9-NEXT: s_lshr_b32 s9, s16, 16
; GFX9-NEXT: s_lshr_b32 s11, s16, 8
; GFX9-NEXT: s_cbranch_execnz .LBB105_4
; GFX9-NEXT: .LBB105_2: ; %cmp.true
@ -15362,9 +15362,9 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX9-NEXT: s_branch .LBB105_5
; GFX9-NEXT: .LBB105_3:
; GFX9-NEXT: ; implicit-def: $sgpr11
; GFX9-NEXT: ; implicit-def: $sgpr10
; GFX9-NEXT: ; implicit-def: $sgpr4
; GFX9-NEXT: ; implicit-def: $sgpr9
; GFX9-NEXT: ; implicit-def: $sgpr4
; GFX9-NEXT: ; implicit-def: $sgpr10
; GFX9-NEXT: ; implicit-def: $sgpr8
; GFX9-NEXT: ; implicit-def: $sgpr5
; GFX9-NEXT: s_branch .LBB105_2
@ -15372,8 +15372,8 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX9-NEXT: v_mov_b32_e32 v8, s16
; GFX9-NEXT: v_mov_b32_e32 v9, s17
; GFX9-NEXT: v_mov_b32_e32 v1, s11
; GFX9-NEXT: v_mov_b32_e32 v2, s10
; GFX9-NEXT: v_mov_b32_e32 v5, s9
; GFX9-NEXT: v_mov_b32_e32 v2, s9
; GFX9-NEXT: v_mov_b32_e32 v5, s10
; GFX9-NEXT: v_mov_b32_e32 v6, s8
; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: v_mov_b32_e32 v3, s4
@ -15392,8 +15392,8 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX11-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 24
; GFX11-NEXT: s_lshr_b32 s5, s1, 16
; GFX11-NEXT: s_lshr_b32 s6, s1, 8
; GFX11-NEXT: s_lshr_b32 s7, s0, 16
; GFX11-NEXT: s_lshr_b32 s7, s1, 8
; GFX11-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-NEXT: s_lshr_b32 s8, s0, 8
; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_vccnz .LBB105_4
@ -15410,16 +15410,16 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX11-NEXT: s_branch .LBB105_5
; GFX11-NEXT: .LBB105_3:
; GFX11-NEXT: ; implicit-def: $sgpr8
; GFX11-NEXT: ; implicit-def: $sgpr7
; GFX11-NEXT: ; implicit-def: $sgpr2
; GFX11-NEXT: ; implicit-def: $sgpr6
; GFX11-NEXT: ; implicit-def: $sgpr2
; GFX11-NEXT: ; implicit-def: $sgpr7
; GFX11-NEXT: ; implicit-def: $sgpr5
; GFX11-NEXT: ; implicit-def: $sgpr3
; GFX11-NEXT: s_branch .LBB105_2
; GFX11-NEXT: .LBB105_4:
; GFX11-NEXT: v_dual_mov_b32 v8, s0 :: v_dual_mov_b32 v9, s1
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s7
; GFX11-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
; GFX11-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s5
; GFX11-NEXT: v_mov_b32_e32 v7, s3
; GFX11-NEXT: v_mov_b32_e32 v3, s2
; GFX11-NEXT: .LBB105_5: ; %end
@ -16493,8 +16493,8 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; VI-NEXT: s_lshr_b64 s[4:5], s[16:17], 24
; VI-NEXT: s_lshr_b32 s8, s17, 24
; VI-NEXT: s_lshr_b32 s5, s17, 16
; VI-NEXT: s_lshr_b32 s9, s17, 8
; VI-NEXT: s_lshr_b32 s10, s16, 16
; VI-NEXT: s_lshr_b32 s10, s17, 8
; VI-NEXT: s_lshr_b32 s9, s16, 16
; VI-NEXT: s_lshr_b32 s11, s16, 8
; VI-NEXT: s_cbranch_execnz .LBB109_4
; VI-NEXT: .LBB109_2: ; %cmp.true
@ -16546,16 +16546,16 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; VI-NEXT: s_setpc_b64 s[30:31]
; VI-NEXT: .LBB109_3:
; VI-NEXT: ; implicit-def: $sgpr11
; VI-NEXT: ; implicit-def: $sgpr10
; VI-NEXT: ; implicit-def: $sgpr4
; VI-NEXT: ; implicit-def: $sgpr9
; VI-NEXT: ; implicit-def: $sgpr4
; VI-NEXT: ; implicit-def: $sgpr10
; VI-NEXT: ; implicit-def: $sgpr5
; VI-NEXT: ; implicit-def: $sgpr8
; VI-NEXT: s_branch .LBB109_2
; VI-NEXT: .LBB109_4:
; VI-NEXT: v_mov_b32_e32 v1, s11
; VI-NEXT: v_mov_b32_e32 v2, s10
; VI-NEXT: v_mov_b32_e32 v5, s9
; VI-NEXT: v_mov_b32_e32 v2, s9
; VI-NEXT: v_mov_b32_e32 v5, s10
; VI-NEXT: v_mov_b32_e32 v7, s8
; VI-NEXT: v_mov_b32_e32 v3, s4
; VI-NEXT: v_mov_b32_e32 v0, s16

94
llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
View File

@ -5548,7 +5548,6 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_mov_b32 s4, 0
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
@ -5557,33 +5556,32 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: v_mov_b32_e32 v5, s4
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mul_i32 s12, s5, 5
; GFX7LESS-NEXT: s_mul_i32 s12, s6, 5
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s14
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s15
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_mov_b32_e32 v9, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v8, v0
; GFX7LESS-NEXT: v_subrev_i32_e32 v6, vcc, s12, v8
; GFX7LESS-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
; GFX7LESS-NEXT: v_mov_b32_e32 v8, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v7, v0
; GFX7LESS-NEXT: v_subrev_i32_e32 v5, vcc, s12, v7
; GFX7LESS-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v8, vcc
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v0, v6
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v7
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v8
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v9
; GFX7LESS-NEXT: v_mov_b32_e32 v0, v5
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v6
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v7
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v8
; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2
@ -5611,39 +5609,37 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
; GFX8-NEXT: s_mov_b32 s4, 0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB9_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
; GFX8-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: s_mul_i32 s12, s5, 5
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s14
; GFX8-NEXT: v_mov_b32_e32 v1, s15
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mul_i32 s12, s6, 5
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: v_mov_b32_e32 v1, s5
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s12, v8
; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
; GFX8-NEXT: v_mov_b32_e32 v0, v6
; GFX8-NEXT: v_mov_b32_e32 v1, v7
; GFX8-NEXT: v_mov_b32_e32 v2, v8
; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: v_mov_b32_e32 v8, v1
; GFX8-NEXT: v_mov_b32_e32 v7, v0
; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s12, v7
; GFX8-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v8, vcc
; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: v_mov_b32_e32 v1, v6
; GFX8-NEXT: v_mov_b32_e32 v2, v7
; GFX8-NEXT: v_mov_b32_e32 v3, v8
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_cbranch_execnz .LBB9_2
@ -5670,39 +5666,37 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
; GFX9-NEXT: s_mov_b32 s4, 0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: v_mov_b32_e32 v5, s4
; GFX9-NEXT: s_mul_i32 s12, s5, 5
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s14
; GFX9-NEXT: v_mov_b32_e32 v1, s15
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mul_i32 s12, s6, 5
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_mov_b32_e32 v9, v1
; GFX9-NEXT: v_mov_b32_e32 v8, v0
; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s12, v8
; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v5, vcc
; GFX9-NEXT: v_mov_b32_e32 v0, v6
; GFX9-NEXT: v_mov_b32_e32 v1, v7
; GFX9-NEXT: v_mov_b32_e32 v2, v8
; GFX9-NEXT: v_mov_b32_e32 v3, v9
; GFX9-NEXT: v_mov_b32_e32 v8, v1
; GFX9-NEXT: v_mov_b32_e32 v7, v0
; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s12, v7
; GFX9-NEXT: v_subbrev_co_u32_e32 v6, vcc, 0, v8, vcc
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: v_mov_b32_e32 v1, v6
; GFX9-NEXT: v_mov_b32_e32 v2, v7
; GFX9-NEXT: v_mov_b32_e32 v3, v8
; GFX9-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_cbranch_execnz .LBB9_2

460
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
View File

@ -1836,22 +1836,22 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_add_f64_e32 v[7:8], v[9:10], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1876,23 +1876,23 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v10, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1907,25 +1907,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_mov_b32_e32 v6, s20
; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v10, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v0
; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, v7
; GFX10-NEXT: v_mov_b32_e32 v1, v8
; GFX10-NEXT: v_mov_b32_e32 v2, v9
; GFX10-NEXT: v_mov_b32_e32 v3, v10
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: v_mov_b32_e32 v1, v7
; GFX10-NEXT: v_mov_b32_e32 v2, v8
; GFX10-NEXT: v_mov_b32_e32 v3, v9
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX10-NEXT: v_mov_b32_e32 v9, v1
; GFX10-NEXT: v_mov_b32_e32 v8, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
@ -1947,25 +1947,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v6
; GFX908-NEXT: v_mov_b32_e32 v1, v7
; GFX908-NEXT: v_mov_b32_e32 v2, v8
; GFX908-NEXT: v_mov_b32_e32 v3, v9
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v8, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB8_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -1977,25 +1977,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v6
; GFX8-NEXT: v_mov_b32_e32 v1, v7
; GFX8-NEXT: v_mov_b32_e32 v2, v8
; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v8, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2007,25 +2007,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v6, s20
; GFX7-NEXT: v_mov_b32_e32 v10, s20
; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v10, v1
; GFX7-NEXT: v_mov_b32_e32 v9, v0
; GFX7-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v7
; GFX7-NEXT: v_mov_b32_e32 v1, v8
; GFX7-NEXT: v_mov_b32_e32 v2, v9
; GFX7-NEXT: v_mov_b32_e32 v3, v10
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2037,27 +2037,27 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: v_mov_b32_e32 v0, s20
; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v6, s6
; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v10, v1
; GFX6-NEXT: v_mov_b32_e32 v9, v0
; GFX6-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v7
; GFX6-NEXT: v_mov_b32_e32 v1, v8
; GFX6-NEXT: v_mov_b32_e32 v2, v9
; GFX6-NEXT: v_mov_b32_e32 v3, v10
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2829,22 +2829,22 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_add_f64_e32 v[7:8], v[9:10], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2869,23 +2869,23 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v10, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2900,25 +2900,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_mov_b32_e32 v6, s20
; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v10, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v0
; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, v7
; GFX10-NEXT: v_mov_b32_e32 v1, v8
; GFX10-NEXT: v_mov_b32_e32 v2, v9
; GFX10-NEXT: v_mov_b32_e32 v3, v10
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: v_mov_b32_e32 v1, v7
; GFX10-NEXT: v_mov_b32_e32 v2, v8
; GFX10-NEXT: v_mov_b32_e32 v3, v9
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX10-NEXT: v_mov_b32_e32 v9, v1
; GFX10-NEXT: v_mov_b32_e32 v8, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB11_1
@ -2931,22 +2931,22 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v4, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
; GFX90A-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v6, s20
; GFX90A-NEXT: v_mov_b32_e32 v10, s20
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[10:11], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_add_f64 v[8:9], v[10:11], v[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[10:11], v[10:11] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2958,25 +2958,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v6
; GFX908-NEXT: v_mov_b32_e32 v1, v7
; GFX908-NEXT: v_mov_b32_e32 v2, v8
; GFX908-NEXT: v_mov_b32_e32 v3, v9
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v8, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2988,25 +2988,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v6
; GFX8-NEXT: v_mov_b32_e32 v1, v7
; GFX8-NEXT: v_mov_b32_e32 v2, v8
; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v8, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@ -3018,25 +3018,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v6, s20
; GFX7-NEXT: v_mov_b32_e32 v10, s20
; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v10, v1
; GFX7-NEXT: v_mov_b32_e32 v9, v0
; GFX7-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v7
; GFX7-NEXT: v_mov_b32_e32 v1, v8
; GFX7-NEXT: v_mov_b32_e32 v2, v9
; GFX7-NEXT: v_mov_b32_e32 v3, v10
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB11_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -3048,27 +3048,27 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: v_mov_b32_e32 v0, s20
; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v6, s6
; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v10, v1
; GFX6-NEXT: v_mov_b32_e32 v9, v0
; GFX6-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v7
; GFX6-NEXT: v_mov_b32_e32 v1, v8
; GFX6-NEXT: v_mov_b32_e32 v2, v9
; GFX6-NEXT: v_mov_b32_e32 v3, v10
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB11_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@ -3090,22 +3090,22 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_add_f64_e32 v[7:8], v[9:10], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -3130,23 +3130,23 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v10, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -3161,25 +3161,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_mov_b32_e32 v6, s20
; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v10, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v0
; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, v7
; GFX10-NEXT: v_mov_b32_e32 v1, v8
; GFX10-NEXT: v_mov_b32_e32 v2, v9
; GFX10-NEXT: v_mov_b32_e32 v3, v10
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: v_mov_b32_e32 v0, v6
; GFX10-NEXT: v_mov_b32_e32 v1, v7
; GFX10-NEXT: v_mov_b32_e32 v2, v8
; GFX10-NEXT: v_mov_b32_e32 v3, v9
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX10-NEXT: v_mov_b32_e32 v9, v1
; GFX10-NEXT: v_mov_b32_e32 v8, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
@ -3201,25 +3201,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v6
; GFX908-NEXT: v_mov_b32_e32 v1, v7
; GFX908-NEXT: v_mov_b32_e32 v2, v8
; GFX908-NEXT: v_mov_b32_e32 v3, v9
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v8, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -3231,25 +3231,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v6
; GFX8-NEXT: v_mov_b32_e32 v1, v7
; GFX8-NEXT: v_mov_b32_e32 v2, v8
; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v8, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@ -3261,25 +3261,25 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v6, s20
; GFX7-NEXT: v_mov_b32_e32 v10, s20
; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v10, v1
; GFX7-NEXT: v_mov_b32_e32 v9, v0
; GFX7-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v7
; GFX7-NEXT: v_mov_b32_e32 v1, v8
; GFX7-NEXT: v_mov_b32_e32 v2, v9
; GFX7-NEXT: v_mov_b32_e32 v3, v10
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB12_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -3291,27 +3291,27 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: v_mov_b32_e32 v0, s20
; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v6, s6
; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v10, v1
; GFX6-NEXT: v_mov_b32_e32 v9, v0
; GFX6-NEXT: v_add_f64 v[7:8], v[9:10], v[4:5]
; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v7
; GFX6-NEXT: v_mov_b32_e32 v1, v8
; GFX6-NEXT: v_mov_b32_e32 v2, v9
; GFX6-NEXT: v_mov_b32_e32 v3, v10
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB12_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end

459
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
View File

@ -1174,28 +1174,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v2, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
; GFX12-NEXT: v_max_num_f64_e32 v[7:8], v[0:1], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
; GFX12-NEXT: v_mov_b32_e32 v0, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1218,29 +1217,28 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v2, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1272,29 +1270,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: v_mov_b32_e32 v3, v5
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -1304,29 +1300,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX8-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB5_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@ -1964,28 +1958,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v2, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
; GFX12-NEXT: v_max_num_f64_e32 v[7:8], v[0:1], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
; GFX12-NEXT: v_mov_b32_e32 v0, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2008,29 +2001,28 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v2, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2042,30 +2034,28 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v6, s20
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX10-NEXT: v_mov_b32_e32 v8, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX10-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v10, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v0
; GFX10-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX10-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, v7
; GFX10-NEXT: v_mov_b32_e32 v1, v8
; GFX10-NEXT: v_mov_b32_e32 v2, v9
; GFX10-NEXT: v_mov_b32_e32 v3, v10
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX10-NEXT: v_mov_b32_e32 v0, v2
; GFX10-NEXT: v_mov_b32_e32 v1, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v4
; GFX10-NEXT: v_mov_b32_e32 v3, v5
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
@ -2076,26 +2066,24 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX90A-NEXT: v_mov_b32_e32 v6, s20
; GFX90A-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX90A-NEXT: v_mov_b32_e32 v8, s20
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[10:11], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
; GFX90A-NEXT: v_max_f64 v[8:9], v[0:1], v[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[10:11], v[10:11] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX90A-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2105,29 +2093,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: v_mov_b32_e32 v3, v5
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB8_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2137,29 +2123,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX8-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2169,29 +2153,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, v0
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v6, s20
; GFX7-NEXT: v_mov_b32_e32 v8, s20
; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v10, v1
; GFX7-NEXT: v_mov_b32_e32 v9, v0
; GFX7-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX7-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v7
; GFX7-NEXT: v_mov_b32_e32 v1, v8
; GFX7-NEXT: v_mov_b32_e32 v2, v9
; GFX7-NEXT: v_mov_b32_e32 v3, v10
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX7-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: v_mov_b32_e32 v2, v4
; GFX7-NEXT: v_mov_b32_e32 v3, v5
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2201,31 +2183,28 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, v0
; GFX6-NEXT: v_mov_b32_e32 v0, s20
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX6-NEXT: v_mov_b32_e32 v2, s20
; GFX6-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
; GFX6-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX6-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v6, s6
; GFX6-NEXT: v_mov_b32_e32 v8, s6
; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v10, v1
; GFX6-NEXT: v_mov_b32_e32 v9, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX6-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX6-NEXT: v_mov_b32_e32 v0, v7
; GFX6-NEXT: v_mov_b32_e32 v1, v8
; GFX6-NEXT: v_mov_b32_e32 v2, v9
; GFX6-NEXT: v_mov_b32_e32 v3, v10
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX6-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: v_mov_b32_e32 v2, v4
; GFX6-NEXT: v_mov_b32_e32 v3, v5
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2245,28 +2224,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v2, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
; GFX12-NEXT: v_max_num_f64_e32 v[7:8], v[0:1], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
; GFX12-NEXT: v_mov_b32_e32 v0, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2289,29 +2267,28 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v2, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX11-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2343,29 +2320,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX908-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: v_mov_b32_e32 v3, v5
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2375,29 +2350,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX8-NEXT: v_max_f64 v[7:8], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end

459
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
View File

@ -1174,28 +1174,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v2, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
; GFX12-NEXT: v_mov_b32_e32 v0, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1218,29 +1217,28 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v2, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -1272,29 +1270,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: v_mov_b32_e32 v3, v5
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -1304,29 +1300,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB5_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@ -1964,28 +1958,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v2, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
; GFX12-NEXT: v_mov_b32_e32 v0, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2008,29 +2001,28 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v2, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2042,30 +2034,28 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v6, s20
; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX10-NEXT: v_mov_b32_e32 v8, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX10-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v10, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v0
; GFX10-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX10-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, v7
; GFX10-NEXT: v_mov_b32_e32 v1, v8
; GFX10-NEXT: v_mov_b32_e32 v2, v9
; GFX10-NEXT: v_mov_b32_e32 v3, v10
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX10-NEXT: v_mov_b32_e32 v0, v2
; GFX10-NEXT: v_mov_b32_e32 v1, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v4
; GFX10-NEXT: v_mov_b32_e32 v3, v5
; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
@ -2076,26 +2066,24 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX90A-NEXT: v_mov_b32_e32 v6, s20
; GFX90A-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX90A-NEXT: v_mov_b32_e32 v8, s20
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[10:11], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
; GFX90A-NEXT: v_min_f64 v[8:9], v[0:1], v[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[10:11], v[10:11] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX90A-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2105,29 +2093,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: v_mov_b32_e32 v3, v5
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB8_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2137,29 +2123,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2169,29 +2153,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, v0
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v6, s20
; GFX7-NEXT: v_mov_b32_e32 v8, s20
; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v10, v1
; GFX7-NEXT: v_mov_b32_e32 v9, v0
; GFX7-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX7-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v7
; GFX7-NEXT: v_mov_b32_e32 v1, v8
; GFX7-NEXT: v_mov_b32_e32 v2, v9
; GFX7-NEXT: v_mov_b32_e32 v3, v10
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX7-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: v_mov_b32_e32 v2, v4
; GFX7-NEXT: v_mov_b32_e32 v3, v5
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2201,31 +2183,28 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v2, v0
; GFX6-NEXT: v_mov_b32_e32 v0, s20
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX6-NEXT: v_mov_b32_e32 v2, s20
; GFX6-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
; GFX6-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX6-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mov_b32_e32 v6, s6
; GFX6-NEXT: v_mov_b32_e32 v8, s6
; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v10, v1
; GFX6-NEXT: v_mov_b32_e32 v9, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX6-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX6-NEXT: v_mov_b32_e32 v0, v7
; GFX6-NEXT: v_mov_b32_e32 v1, v8
; GFX6-NEXT: v_mov_b32_e32 v2, v9
; GFX6-NEXT: v_mov_b32_e32 v3, v10
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX6-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX6-NEXT: v_mov_b32_e32 v0, v2
; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: v_mov_b32_e32 v2, v4
; GFX6-NEXT: v_mov_b32_e32 v3, v5
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2245,28 +2224,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: v_mov_b32_e32 v2, s16
; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[9:10], v[9:10]
; GFX12-NEXT: v_min_num_f64_e32 v[7:8], v[0:1], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX12-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
; GFX12-NEXT: v_mov_b32_e32 v0, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX12-NEXT: v_mov_b32_e32 v3, v5
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu 0xfffe
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2289,29 +2267,28 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: v_mov_b32_e32 v2, s16
; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX11-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8
; GFX11-NEXT: v_dual_mov_b32 v2, v9 :: v_dual_mov_b32 v3, v10
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4
; GFX11-NEXT: v_mov_b32_e32 v3, v5
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[9:10]
; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@ -2343,29 +2320,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v10, v1
; GFX908-NEXT: v_mov_b32_e32 v9, v0
; GFX908-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX908-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v7
; GFX908-NEXT: v_mov_b32_e32 v1, v8
; GFX908-NEXT: v_mov_b32_e32 v2, v9
; GFX908-NEXT: v_mov_b32_e32 v3, v10
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v2
; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: v_mov_b32_e32 v3, v5
; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@ -2375,29 +2350,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v10, v1
; GFX8-NEXT: v_mov_b32_e32 v9, v0
; GFX8-NEXT: v_max_f64 v[0:1], v[9:10], v[9:10]
; GFX8-NEXT: v_min_f64 v[7:8], v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v7
; GFX8-NEXT: v_mov_b32_e32 v1, v8
; GFX8-NEXT: v_mov_b32_e32 v2, v9
; GFX8-NEXT: v_mov_b32_e32 v3, v10
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end

126
llvm/test/CodeGen/AMDGPU/div_v2i128.ll
View File

@ -619,43 +619,43 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: s_mov_b64 s[8:9], 0
; GISEL-NEXT: v_ashrrev_i32_e32 v18, 31, v7
; GISEL-NEXT: v_ashrrev_i32_e32 v19, 31, v15
; GISEL-NEXT: v_mov_b32_e32 v10, 0x7f
; GISEL-NEXT: v_mov_b32_e32 v11, 0
; GISEL-NEXT: v_mov_b32_e32 v16, 0x7f
; GISEL-NEXT: v_mov_b32_e32 v17, 0
; GISEL-NEXT: v_xor_b32_e32 v0, v18, v4
; GISEL-NEXT: v_xor_b32_e32 v1, v18, v5
; GISEL-NEXT: v_xor_b32_e32 v2, v18, v6
; GISEL-NEXT: v_xor_b32_e32 v3, v18, v7
; GISEL-NEXT: v_xor_b32_e32 v4, v19, v12
; GISEL-NEXT: v_xor_b32_e32 v5, v19, v13
; GISEL-NEXT: v_xor_b32_e32 v14, v19, v14
; GISEL-NEXT: v_xor_b32_e32 v15, v19, v15
; GISEL-NEXT: v_xor_b32_e32 v12, v19, v14
; GISEL-NEXT: v_xor_b32_e32 v13, v19, v15
; GISEL-NEXT: v_sub_i32_e32 v6, vcc, v0, v18
; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v1, v18, vcc
; GISEL-NEXT: v_sub_i32_e64 v20, s[4:5], v4, v19
; GISEL-NEXT: v_subb_u32_e64 v21, s[4:5], v5, v19, s[4:5]
; GISEL-NEXT: v_subb_u32_e32 v12, vcc, v2, v18, vcc
; GISEL-NEXT: v_subb_u32_e32 v13, vcc, v3, v18, vcc
; GISEL-NEXT: v_subb_u32_e64 v4, vcc, v14, v19, s[4:5]
; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v15, v19, vcc
; GISEL-NEXT: v_ffbh_u32_e32 v14, v21
; GISEL-NEXT: v_ffbh_u32_e32 v15, v20
; GISEL-NEXT: v_ffbh_u32_e32 v16, v7
; GISEL-NEXT: v_ffbh_u32_e32 v17, v6
; GISEL-NEXT: v_subb_u32_e32 v10, vcc, v2, v18, vcc
; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v3, v18, vcc
; GISEL-NEXT: v_subb_u32_e64 v4, vcc, v12, v19, s[4:5]
; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v13, v19, vcc
; GISEL-NEXT: v_ffbh_u32_e32 v12, v21
; GISEL-NEXT: v_ffbh_u32_e32 v13, v20
; GISEL-NEXT: v_ffbh_u32_e32 v14, v7
; GISEL-NEXT: v_ffbh_u32_e32 v15, v6
; GISEL-NEXT: v_or_b32_e32 v0, v20, v4
; GISEL-NEXT: v_or_b32_e32 v1, v21, v5
; GISEL-NEXT: v_or_b32_e32 v2, v6, v12
; GISEL-NEXT: v_or_b32_e32 v3, v7, v13
; GISEL-NEXT: v_add_i32_e32 v15, vcc, 32, v15
; GISEL-NEXT: v_or_b32_e32 v2, v6, v10
; GISEL-NEXT: v_or_b32_e32 v3, v7, v11
; GISEL-NEXT: v_add_i32_e32 v13, vcc, 32, v13
; GISEL-NEXT: v_ffbh_u32_e32 v26, v5
; GISEL-NEXT: v_ffbh_u32_e32 v27, v4
; GISEL-NEXT: v_add_i32_e32 v17, vcc, 32, v17
; GISEL-NEXT: v_ffbh_u32_e32 v28, v13
; GISEL-NEXT: v_ffbh_u32_e32 v29, v12
; GISEL-NEXT: v_add_i32_e32 v15, vcc, 32, v15
; GISEL-NEXT: v_ffbh_u32_e32 v28, v11
; GISEL-NEXT: v_ffbh_u32_e32 v29, v10
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GISEL-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[2:3]
; GISEL-NEXT: v_min_u32_e32 v0, v14, v15
; GISEL-NEXT: v_min_u32_e32 v0, v12, v13
; GISEL-NEXT: v_add_i32_e64 v1, s[6:7], 32, v27
; GISEL-NEXT: v_min_u32_e32 v2, v16, v17
; GISEL-NEXT: v_min_u32_e32 v2, v14, v15
; GISEL-NEXT: v_add_i32_e64 v3, s[6:7], 32, v29
; GISEL-NEXT: v_add_i32_e64 v0, s[6:7], 64, v0
; GISEL-NEXT: v_min_u32_e32 v1, v26, v1
@ -665,32 +665,32 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, s[4:5]
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
; GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
; GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v0, v1
; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, vcc
; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], 0, 0, s[4:5]
; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5]
; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[10:11]
; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[16:17]
; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
; GISEL-NEXT: v_xor_b32_e32 v10, 0x7f, v2
; GISEL-NEXT: v_xor_b32_e32 v12, 0x7f, v2
; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[0:1]
; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
; GISEL-NEXT: v_or_b32_e32 v10, v10, v0
; GISEL-NEXT: v_or_b32_e32 v11, v3, v1
; GISEL-NEXT: v_or_b32_e32 v12, v12, v0
; GISEL-NEXT: v_or_b32_e32 v13, v3, v1
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GISEL-NEXT: v_cndmask_b32_e32 v15, v16, v15, vcc
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GISEL-NEXT: v_or_b32_e32 v11, v14, v15
; GISEL-NEXT: v_and_b32_e32 v14, 1, v11
; GISEL-NEXT: v_or_b32_e32 v10, v11, v10
; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GISEL-NEXT: v_or_b32_e32 v13, v14, v15
; GISEL-NEXT: v_and_b32_e32 v14, 1, v13
; GISEL-NEXT: v_or_b32_e32 v12, v13, v12
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v14
; GISEL-NEXT: v_cndmask_b32_e64 v14, v6, 0, vcc
; GISEL-NEXT: v_and_b32_e32 v16, 1, v10
; GISEL-NEXT: v_and_b32_e32 v16, 1, v12
; GISEL-NEXT: v_cndmask_b32_e64 v15, v7, 0, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v10, v12, 0, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v11, v13, 0, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v12, v10, 0, vcc
; GISEL-NEXT: v_cndmask_b32_e64 v13, v11, 0, vcc
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], s[4:5]
@ -703,22 +703,22 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_addc_u32_e64 v28, vcc, 0, v0, s[4:5]
; GISEL-NEXT: v_addc_u32_e32 v29, vcc, 0, v1, vcc
; GISEL-NEXT: v_add_i32_e64 v14, s[4:5], v30, v2
; GISEL-NEXT: v_sub_i32_e64 v10, s[4:5], 64, v30
; GISEL-NEXT: v_sub_i32_e64 v12, s[4:5], 64, v30
; GISEL-NEXT: v_lshl_b64 v[0:1], v[6:7], v30
; GISEL-NEXT: v_lshl_b64 v[2:3], v[12:13], v30
; GISEL-NEXT: v_lshl_b64 v[2:3], v[10:11], v30
; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
; GISEL-NEXT: v_lshr_b64 v[10:11], v[6:7], v10
; GISEL-NEXT: v_lshr_b64 v[12:13], v[6:7], v12
; GISEL-NEXT: v_lshl_b64 v[16:17], v[6:7], v14
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v30
; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v0, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v1, vcc
; GISEL-NEXT: v_or_b32_e32 v0, v10, v2
; GISEL-NEXT: v_or_b32_e32 v1, v11, v3
; GISEL-NEXT: v_or_b32_e32 v0, v12, v2
; GISEL-NEXT: v_or_b32_e32 v1, v13, v3
; GISEL-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v30
; GISEL-NEXT: v_cndmask_b32_e32 v10, v0, v12, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v11, v1, v13, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v12, v0, v10, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v13, v1, v11, vcc
; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
; GISEL-NEXT: v_mov_b32_e32 v0, s8
; GISEL-NEXT: v_mov_b32_e32 v1, s9
@ -730,26 +730,26 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
; GISEL-NEXT: v_add_i32_e32 v32, vcc, 0xffffffc0, v26
; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 64, v26
; GISEL-NEXT: v_lshr_b64 v[0:1], v[12:13], v26
; GISEL-NEXT: v_lshr_b64 v[0:1], v[10:11], v26
; GISEL-NEXT: v_lshr_b64 v[2:3], v[6:7], v26
; GISEL-NEXT: s_mov_b64 s[4:5], 0
; GISEL-NEXT: v_add_i32_e32 v30, vcc, -1, v20
; GISEL-NEXT: v_addc_u32_e32 v31, vcc, -1, v21, vcc
; GISEL-NEXT: v_lshl_b64 v[16:17], v[12:13], v16
; GISEL-NEXT: v_lshr_b64 v[12:13], v[12:13], v32
; GISEL-NEXT: v_lshl_b64 v[16:17], v[10:11], v16
; GISEL-NEXT: v_lshr_b64 v[10:11], v[10:11], v32
; GISEL-NEXT: v_addc_u32_e32 v32, vcc, -1, v4, vcc
; GISEL-NEXT: v_addc_u32_e32 v33, vcc, -1, v5, vcc
; GISEL-NEXT: s_mov_b64 s[6:7], s[4:5]
; GISEL-NEXT: v_or_b32_e32 v2, v2, v16
; GISEL-NEXT: v_or_b32_e32 v3, v3, v17
; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v26
; GISEL-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v13, v3, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v0, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v1, vcc
; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v26
; GISEL-NEXT: v_cndmask_b32_e32 v12, v2, v6, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v13, v3, v7, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v10, v2, v6, vcc
; GISEL-NEXT: v_cndmask_b32_e32 v11, v3, v7, vcc
; GISEL-NEXT: v_mov_b32_e32 v7, 0
; GISEL-NEXT: v_mov_b32_e32 v0, s4
; GISEL-NEXT: v_mov_b32_e32 v1, s5
@ -757,20 +757,20 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_mov_b32_e32 v3, s7
; GISEL-NEXT: .LBB0_9: ; %udiv-do-while
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: v_lshl_b64 v[2:3], v[12:13], 1
; GISEL-NEXT: v_lshl_b64 v[2:3], v[10:11], 1
; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v13
; GISEL-NEXT: v_lshrrev_b32_e32 v34, 31, v11
; GISEL-NEXT: v_lshl_b64 v[12:13], v[14:15], 1
; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v11
; GISEL-NEXT: v_lshrrev_b32_e32 v34, 31, v13
; GISEL-NEXT: v_lshl_b64 v[10:11], v[14:15], 1
; GISEL-NEXT: v_lshl_b64 v[12:13], v[12:13], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v14, 31, v15
; GISEL-NEXT: v_add_i32_e32 v26, vcc, -1, v26
; GISEL-NEXT: v_addc_u32_e32 v27, vcc, -1, v27, vcc
; GISEL-NEXT: v_or_b32_e32 v16, v16, v6
; GISEL-NEXT: v_or_b32_e32 v2, v2, v34
; GISEL-NEXT: v_or_b32_e32 v10, v10, v14
; GISEL-NEXT: v_or_b32_e32 v14, v0, v12
; GISEL-NEXT: v_or_b32_e32 v15, v1, v13
; GISEL-NEXT: v_or_b32_e32 v12, v12, v14
; GISEL-NEXT: v_or_b32_e32 v14, v0, v10
; GISEL-NEXT: v_or_b32_e32 v15, v1, v11
; GISEL-NEXT: v_addc_u32_e32 v28, vcc, -1, v28, vcc
; GISEL-NEXT: v_addc_u32_e32 v29, vcc, -1, v29, vcc
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v30, v2
@ -783,14 +783,14 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_ashrrev_i32_e32 v0, 31, v6
; GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GISEL-NEXT: v_and_b32_e32 v6, 1, v0
; GISEL-NEXT: v_and_b32_e32 v12, v0, v20
; GISEL-NEXT: v_and_b32_e32 v13, v0, v21
; GISEL-NEXT: v_and_b32_e32 v10, v0, v20
; GISEL-NEXT: v_and_b32_e32 v11, v0, v21
; GISEL-NEXT: v_and_b32_e32 v34, v0, v4
; GISEL-NEXT: v_and_b32_e32 v35, v0, v5
; GISEL-NEXT: v_mov_b32_e32 v0, v6
; GISEL-NEXT: v_mov_b32_e32 v1, v7
; GISEL-NEXT: v_sub_i32_e32 v12, vcc, v2, v12
; GISEL-NEXT: v_subb_u32_e32 v13, vcc, v3, v13, vcc
; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v2, v10
; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v3, v11, vcc
; GISEL-NEXT: v_subb_u32_e32 v16, vcc, v16, v34, vcc
; GISEL-NEXT: v_subb_u32_e32 v17, vcc, v17, v35, vcc
; GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
@ -800,9 +800,9 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: .LBB0_11: ; %Flow11
; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
; GISEL-NEXT: v_lshl_b64 v[2:3], v[14:15], 1
; GISEL-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
; GISEL-NEXT: v_lshl_b64 v[12:13], v[12:13], 1
; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v15
; GISEL-NEXT: v_or_b32_e32 v10, v10, v4
; GISEL-NEXT: v_or_b32_e32 v12, v12, v4
; GISEL-NEXT: v_or_b32_e32 v14, v0, v2
; GISEL-NEXT: v_or_b32_e32 v15, v1, v3
; GISEL-NEXT: .LBB0_12: ; %Flow12
@ -815,8 +815,8 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; GISEL-NEXT: v_xor_b32_e32 v6, v9, v3
; GISEL-NEXT: v_xor_b32_e32 v4, v14, v7
; GISEL-NEXT: v_xor_b32_e32 v5, v15, v7
; GISEL-NEXT: v_xor_b32_e32 v8, v10, v7
; GISEL-NEXT: v_xor_b32_e32 v9, v11, v7
; GISEL-NEXT: v_xor_b32_e32 v8, v12, v7
; GISEL-NEXT: v_xor_b32_e32 v9, v13, v7
; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v4, v7

26
llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
View File

@ -20,7 +20,8 @@ define i128 @fptosi_f64_to_i128(double %x) {
; SDAG-NEXT: s_cbranch_execz .LBB0_10
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
@ -386,7 +387,8 @@ define i128 @fptoui_f64_to_i128(double %x) {
; SDAG-NEXT: s_cbranch_execz .LBB1_10
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
@ -749,9 +751,10 @@ define i128 @fptosi_f32_to_i128(float %x) {
; SDAG-NEXT: s_and_saveexec_b64 s[8:9], vcc
; SDAG-NEXT: s_cbranch_execz .LBB2_10
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
@ -1100,9 +1103,10 @@ define i128 @fptoui_f32_to_i128(float %x) {
; SDAG-NEXT: s_and_saveexec_b64 s[8:9], vcc
; SDAG-NEXT: s_cbranch_execz .LBB3_10
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
@ -1489,9 +1493,10 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
; SDAG-NEXT: s_and_saveexec_b64 s[8:9], vcc
; SDAG-NEXT: s_cbranch_execz .LBB6_10
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
@ -1836,9 +1841,10 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
; SDAG-NEXT: s_and_saveexec_b64 s[8:9], vcc
; SDAG-NEXT: s_cbranch_execz .LBB7_10
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v6, vcc

76
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
View File

@ -7575,15 +7575,13 @@ define double @global_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
; GFX7-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v5, v3
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v8
; GFX7-NEXT: v_mov_b32_e32 v1, v9
@ -7593,7 +7591,9 @@ define double @global_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB38_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -7609,15 +7609,13 @@ define double @global_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
; GFX6-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64
; GFX6-NEXT: v_mov_b32_e32 v5, v3
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v8
@ -7628,7 +7626,9 @@ define double @global_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB38_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@ -7809,15 +7809,13 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:2040
; GFX7-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:2040
; GFX7-NEXT: v_mov_b32_e32 v5, v3
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v8
; GFX7-NEXT: v_mov_b32_e32 v1, v9
@ -7827,7 +7825,9 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB39_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -7843,15 +7843,13 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:2040
; GFX6-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:2040
; GFX6-NEXT: v_mov_b32_e32 v5, v3
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v8
@ -7862,7 +7860,9 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB39_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@ -8039,34 +8039,32 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
; GFX7-NEXT: v_mov_b32_e32 v7, v1
; GFX7-NEXT: v_mov_b32_e32 v6, v0
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
; GFX7-NEXT: v_add_i32_e32 v6, vcc, 0xfffff800, v6
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_add_i32_e32 v10, vcc, 0xfffff800, v0
; GFX7-NEXT: v_mov_b32_e32 v5, v3
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: v_addc_u32_e32 v7, vcc, -1, v7, vcc
; GFX7-NEXT: v_addc_u32_e32 v11, vcc, -1, v1, vcc
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v8
; GFX7-NEXT: v_mov_b32_e32 v1, v9
; GFX7-NEXT: v_mov_b32_e32 v2, v10
; GFX7-NEXT: v_mov_b32_e32 v3, v11
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[10:11], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB40_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -8077,35 +8075,33 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
; GFX6-NEXT: v_mov_b32_e32 v7, v1
; GFX6-NEXT: v_mov_b32_e32 v6, v0
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffff800, v6
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_add_i32_e32 v10, vcc, 0xfffff800, v0
; GFX6-NEXT: v_mov_b32_e32 v5, v3
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: v_addc_u32_e32 v7, vcc, -1, v7, vcc
; GFX6-NEXT: v_addc_u32_e32 v11, vcc, -1, v1, vcc
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], v[4:5]
; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v8
; GFX6-NEXT: v_mov_b32_e32 v1, v9
; GFX6-NEXT: v_mov_b32_e32 v2, v10
; GFX6-NEXT: v_mov_b32_e32 v3, v11
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[10:11], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB40_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end

47
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
View File

@ -4203,25 +4203,25 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
; GFX7-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v[4:5], s[4:7], 0 addr64
; GFX7-NEXT: v_max_f64 v[10:11], v[2:3], v[2:3]
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
; GFX7-NEXT: v_max_f64 v[8:9], v[0:1], v[6:7]
; GFX7-NEXT: v_mov_b32_e32 v0, v8
; GFX7-NEXT: v_mov_b32_e32 v1, v9
; GFX7-NEXT: v_mov_b32_e32 v2, v10
; GFX7-NEXT: v_mov_b32_e32 v3, v11
; GFX7-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
; GFX7-NEXT: v_max_f64 v[6:7], v[0:1], v[10:11]
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -4237,26 +4237,25 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
; GFX6-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v[4:5], s[4:7], 0 addr64
; GFX6-NEXT: v_max_f64 v[10:11], v[2:3], v[2:3]
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
; GFX6-NEXT: v_max_f64 v[8:9], v[0:1], v[6:7]
; GFX6-NEXT: v_mov_b32_e32 v0, v8
; GFX6-NEXT: v_mov_b32_e32 v1, v9
; GFX6-NEXT: v_mov_b32_e32 v2, v10
; GFX6-NEXT: v_mov_b32_e32 v3, v11
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
; GFX6-NEXT: v_max_f64 v[6:7], v[0:1], v[10:11]
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB24_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end

47
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
View File

@ -4203,25 +4203,25 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
; GFX7-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v[4:5], s[4:7], 0 addr64
; GFX7-NEXT: v_max_f64 v[10:11], v[2:3], v[2:3]
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
; GFX7-NEXT: v_min_f64 v[8:9], v[0:1], v[6:7]
; GFX7-NEXT: v_mov_b32_e32 v0, v8
; GFX7-NEXT: v_mov_b32_e32 v1, v9
; GFX7-NEXT: v_mov_b32_e32 v2, v10
; GFX7-NEXT: v_mov_b32_e32 v3, v11
; GFX7-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
; GFX7-NEXT: v_min_f64 v[6:7], v[0:1], v[10:11]
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -4237,26 +4237,25 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
; GFX6-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v[4:5], s[4:7], 0 addr64
; GFX6-NEXT: v_max_f64 v[10:11], v[2:3], v[2:3]
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[10:11], v[10:11]
; GFX6-NEXT: v_min_f64 v[8:9], v[0:1], v[6:7]
; GFX6-NEXT: v_mov_b32_e32 v0, v8
; GFX6-NEXT: v_mov_b32_e32 v1, v9
; GFX6-NEXT: v_mov_b32_e32 v2, v10
; GFX6-NEXT: v_mov_b32_e32 v3, v11
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
; GFX6-NEXT: v_min_f64 v[6:7], v[0:1], v[10:11]
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB24_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end

76
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
View File

@ -3913,15 +3913,13 @@ define double @global_agent_atomic_fsub_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
; GFX7-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64
; GFX7-NEXT: v_mov_b32_e32 v5, v3
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v8
; GFX7-NEXT: v_mov_b32_e32 v1, v9
@ -3931,7 +3929,9 @@ define double @global_agent_atomic_fsub_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB16_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -3947,15 +3947,13 @@ define double @global_agent_atomic_fsub_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
; GFX6-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64
; GFX6-NEXT: v_mov_b32_e32 v5, v3
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v8
@ -3966,7 +3964,9 @@ define double @global_agent_atomic_fsub_ret_f64(ptr addrspace(1) %ptr, double %v
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB16_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@ -4165,15 +4165,13 @@ define double @global_agent_atomic_fsub_ret_f64__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:2040
; GFX7-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:2040
; GFX7-NEXT: v_mov_b32_e32 v5, v3
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v8
; GFX7-NEXT: v_mov_b32_e32 v1, v9
@ -4183,7 +4181,9 @@ define double @global_agent_atomic_fsub_ret_f64__offset12b_pos(ptr addrspace(1)
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB17_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -4199,15 +4199,13 @@ define double @global_agent_atomic_fsub_ret_f64__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:2040
; GFX6-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64 offset:2040
; GFX6-NEXT: v_mov_b32_e32 v5, v3
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v8
@ -4218,7 +4216,9 @@ define double @global_agent_atomic_fsub_ret_f64__offset12b_pos(ptr addrspace(1)
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB17_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@ -4413,34 +4413,32 @@ define double @global_agent_atomic_fsub_ret_f64__offset12b_neg(ptr addrspace(1)
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_movk_i32 s4, 0xf800
; GFX7-NEXT: v_mov_b32_e32 v7, v1
; GFX7-NEXT: v_mov_b32_e32 v6, v0
; GFX7-NEXT: s_mov_b32 s5, -1
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
; GFX7-NEXT: v_add_i32_e32 v6, vcc, 0xfffff800, v6
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: v_add_i32_e32 v10, vcc, 0xfffff800, v0
; GFX7-NEXT: v_mov_b32_e32 v5, v3
; GFX7-NEXT: v_mov_b32_e32 v4, v2
; GFX7-NEXT: v_addc_u32_e32 v7, vcc, -1, v7, vcc
; GFX7-NEXT: v_addc_u32_e32 v11, vcc, -1, v1, vcc
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
; GFX7-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v11, v1
; GFX7-NEXT: v_mov_b32_e32 v10, v0
; GFX7-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v8
; GFX7-NEXT: v_mov_b32_e32 v1, v9
; GFX7-NEXT: v_mov_b32_e32 v2, v10
; GFX7-NEXT: v_mov_b32_e32 v3, v11
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], -v[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[10:11], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB18_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@ -4451,35 +4449,33 @@ define double @global_agent_atomic_fsub_ret_f64__offset12b_neg(ptr addrspace(1)
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_movk_i32 s4, 0xf800
; GFX6-NEXT: v_mov_b32_e32 v7, v1
; GFX6-NEXT: v_mov_b32_e32 v6, v0
; GFX6-NEXT: s_mov_b32 s5, -1
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
; GFX6-NEXT: v_add_i32_e32 v6, vcc, 0xfffff800, v6
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64
; GFX6-NEXT: v_add_i32_e32 v10, vcc, 0xfffff800, v0
; GFX6-NEXT: v_mov_b32_e32 v5, v3
; GFX6-NEXT: v_mov_b32_e32 v4, v2
; GFX6-NEXT: v_addc_u32_e32 v7, vcc, -1, v7, vcc
; GFX6-NEXT: v_addc_u32_e32 v11, vcc, -1, v1, vcc
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v11, v1
; GFX6-NEXT: v_mov_b32_e32 v10, v0
; GFX6-NEXT: v_add_f64 v[8:9], v[10:11], -v[4:5]
; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], -v[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, v8
; GFX6-NEXT: v_mov_b32_e32 v1, v9
; GFX6-NEXT: v_mov_b32_e32 v2, v10
; GFX6-NEXT: v_mov_b32_e32 v3, v11
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[10:11], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end

721
llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
View File

File diff suppressed because it is too large Load Diff

3
llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
View File

@ -76,13 +76,12 @@ define amdgpu_kernel void @v_round_f64(ptr addrspace(1) %out, ptr addrspace(1) %
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_movk_i32 s4, 0xfc01
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_mov_b32 s3, 0xfffff
; SI-NEXT: v_mov_b32_e32 v8, 0x3ff00000
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_bfe_u32 v4, v3, 20, 11
; SI-NEXT: v_add_i32_e32 v6, vcc, s4, v4
; SI-NEXT: v_add_i32_e32 v6, vcc, 0xfffffc01, v4
; SI-NEXT: v_lshr_b64 v[4:5], s[2:3], v6
; SI-NEXT: v_and_b32_e32 v7, 0x80000000, v3
; SI-NEXT: v_bfi_b32 v5, v5, 0, v3

2
llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
View File

@ -51,7 +51,7 @@ define void @issue63986(i64 %0, i64 %idxprom, ptr inreg %ptr) {
; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s4, v2
; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v3, v7, vcc
; CHECK-NEXT: s_add_u32 s4, s4, 1
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: s_addc_u32 s5, 0, s5
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: flat_store_byte v[6:7], v10
; CHECK-NEXT: ; %bb.7:

4
llvm/test/CodeGen/AMDGPU/peephole-opt-regseq-removal.mir
View File

@ -113,7 +113,7 @@ body: |
; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GCN-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]].sub0
; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
; GCN-NEXT: $vgpr0 = COPY [[COPY3]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1
@ -135,7 +135,7 @@ body: |
; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GCN-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY [[REG_SEQUENCE]]
; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]].sub0
; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub0
; GCN-NEXT: $vgpr0 = COPY [[COPY3]]
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = COPY $vgpr1

6
llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
View File

@ -495,8 +495,7 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX900-NEXT: v_mov_b32_e32 v1, s35
; GFX900-NEXT: v_add_co_u32_e32 v0, vcc, s34, v0
; GFX900-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX900-NEXT: s_movk_i32 s0, 0x5000
; GFX900-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
; GFX900-NEXT: v_add_co_u32_e32 v0, vcc, 0x5000, v0
; GFX900-NEXT: v_mov_b32_e32 v4, 0
; GFX900-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX900-NEXT: v_mov_b32_e32 v5, 0
@ -718,8 +717,7 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX90A-NEXT: v_mov_b32_e32 v2, s35
; GFX90A-NEXT: v_add_co_u32_e32 v1, vcc, s34, v1
; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX90A-NEXT: s_movk_i32 s0, 0x5000
; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, s0, v1
; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x5000, v1
; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], 0, 0
; GFX90A-NEXT: v_mov_b32_e32 v1, 0x7f

16
llvm/test/CodeGen/ARM/llvm.exp10.ll
View File

@ -189,12 +189,13 @@ define <3 x float> @exp10_v3f32(<3 x float> %x) {
; CHECK-NEXT: mov r6, r0
; CHECK-NEXT: mov r0, r4
; CHECK-NEXT: bl exp10f
; CHECK-NEXT: mov r4, r0
; CHECK-NEXT: vmov s17, r0
; CHECK-NEXT: mov r0, r5
; CHECK-NEXT: bl exp10f
; CHECK-NEXT: vmov s16, r0
; CHECK-NEXT: mov r1, r4
; CHECK-NEXT: vmov s18, r6
; CHECK-NEXT: vmov r0, r1, d8
; CHECK-NEXT: vmov r2, r3, d9
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: pop {r4, r5, r6, pc}
@ -207,7 +208,6 @@ define <4 x float> @exp10_v4f32(<4 x float> %x) {
; CHECK: @ %bb.0:
; CHECK-NEXT: push {r4, r5, r6, r7, lr}
; CHECK-NEXT: sub sp, #4
; CHECK-NEXT: vpush {d8, d9}
; CHECK-NEXT: mov r6, r0
; CHECK-NEXT: mov r0, r1
; CHECK-NEXT: mov r4, r3
@ -216,17 +216,15 @@ define <4 x float> @exp10_v4f32(<4 x float> %x) {
; CHECK-NEXT: mov r7, r0
; CHECK-NEXT: mov r0, r4
; CHECK-NEXT: bl exp10f
; CHECK-NEXT: vmov s19, r0
; CHECK-NEXT: mov r4, r0
; CHECK-NEXT: mov r0, r5
; CHECK-NEXT: bl exp10f
; CHECK-NEXT: vmov s18, r0
; CHECK-NEXT: mov r5, r0
; CHECK-NEXT: mov r0, r6
; CHECK-NEXT: vmov s17, r7
; CHECK-NEXT: bl exp10f
; CHECK-NEXT: vmov s16, r0
; CHECK-NEXT: vmov r2, r3, d9
; CHECK-NEXT: vmov r0, r1, d8
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: mov r1, r7
; CHECK-NEXT: mov r2, r5
; CHECK-NEXT: mov r3, r4
; CHECK-NEXT: add sp, #4
; CHECK-NEXT: pop {r4, r5, r6, r7, pc}
%r = call <4 x float> @llvm.exp10.v4f32(<4 x float> %x)

36
llvm/test/CodeGen/ARM/llvm.frexp.ll
View File

@ -362,33 +362,31 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) {
define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) {
; CHECK-LABEL: test_frexp_v4f32_v4i32_only_use_fract:
; CHECK: @ %bb.0:
; CHECK-NEXT: push {r4, r5, r6, lr}
; CHECK-NEXT: vpush {d8, d9}
; CHECK-NEXT: sub sp, #16
; CHECK-NEXT: mov r5, r1
; CHECK-NEXT: mov r6, r0
; CHECK-NEXT: mov r1, sp
; CHECK-NEXT: mov r0, r3
; CHECK-NEXT: mov r4, r2
; CHECK-NEXT: bl frexpf
; CHECK-NEXT: push {r4, r5, r6, r7, lr}
; CHECK-NEXT: sub sp, #20
; CHECK-NEXT: mov r6, r1
; CHECK-NEXT: add r1, sp, #4
; CHECK-NEXT: vmov s19, r0
; CHECK-NEXT: mov r0, r4
; CHECK-NEXT: mov r7, r0
; CHECK-NEXT: mov r0, r3
; CHECK-NEXT: mov r5, r2
; CHECK-NEXT: bl frexpf
; CHECK-NEXT: add r1, sp, #8
; CHECK-NEXT: vmov s18, r0
; CHECK-NEXT: mov r4, r0
; CHECK-NEXT: mov r0, r5
; CHECK-NEXT: bl frexpf
; CHECK-NEXT: add r1, sp, #12
; CHECK-NEXT: vmov s17, r0
; CHECK-NEXT: mov r5, r0
; CHECK-NEXT: mov r0, r6
; CHECK-NEXT: bl frexpf
; CHECK-NEXT: vmov s16, r0
; CHECK-NEXT: vmov r2, r3, d9
; CHECK-NEXT: vmov r0, r1, d8
; CHECK-NEXT: add sp, #16
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: pop {r4, r5, r6, pc}
; CHECK-NEXT: add r1, sp, #16
; CHECK-NEXT: mov r6, r0
; CHECK-NEXT: mov r0, r7
; CHECK-NEXT: bl frexpf
; CHECK-NEXT: mov r1, r6
; CHECK-NEXT: mov r2, r5
; CHECK-NEXT: mov r3, r4
; CHECK-NEXT: add sp, #20
; CHECK-NEXT: pop {r4, r5, r6, r7, pc}
%result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a)
%result.0 = extractvalue { <4 x float>, <4 x i32> } %result, 0
ret <4 x float> %result.0

1
llvm/test/CodeGen/Mips/no-odd-spreg-msa.ll
View File

@ -97,7 +97,6 @@ entry:
; ALL: lw $[[R0:[0-9]+]], %got(v4f32)(
; ALL: ld.w $w12, 0($[[R0]])
; ALL: move.v $w[[W0:13]], $w12
; NOODDSPREG: move.v $w[[W0:12]], $w13
; ALL: teqi $zero, 1
; ALL-NOT: st.w
; ALL-NOT: ld.w

20
llvm/test/CodeGen/PowerPC/urem-vector-lkk.ll
View File

@ -897,31 +897,31 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) {
; P8LE-NEXT: mfvsrd r6, v2
; P8LE-NEXT: mfvsrd r8, v3
; P8LE-NEXT: ori r3, r3, 51289
; P8LE-NEXT: mffprd r4, f0
; P8LE-NEXT: ori r5, r5, 42889
; P8LE-NEXT: rldic r4, r3, 36, 1
; P8LE-NEXT: mffprd r3, f0
; P8LE-NEXT: rldic r3, r3, 36, 1
; P8LE-NEXT: rldic r5, r5, 35, 1
; P8LE-NEXT: rldicl r7, r6, 63, 1
; P8LE-NEXT: oris r4, r4, 45590
; P8LE-NEXT: oris r3, r3, 45590
; P8LE-NEXT: oris r5, r5, 1603
; P8LE-NEXT: ori r4, r4, 17097
; P8LE-NEXT: ori r3, r3, 17097
; P8LE-NEXT: ori r5, r5, 21445
; P8LE-NEXT: mulhdu r4, r3, r4
; P8LE-NEXT: mulhdu r3, r4, r3
; P8LE-NEXT: mulhdu r5, r7, r5
; P8LE-NEXT: sub r7, r3, r4
; P8LE-NEXT: sub r7, r4, r3
; P8LE-NEXT: rldicl r5, r5, 57, 7
; P8LE-NEXT: rldicl r7, r7, 63, 1
; P8LE-NEXT: mulli r5, r5, 654
; P8LE-NEXT: add r4, r7, r4
; P8LE-NEXT: add r3, r7, r3
; P8LE-NEXT: lis r7, -16037
; P8LE-NEXT: ori r7, r7, 28749
; P8LE-NEXT: rldicl r4, r4, 60, 4
; P8LE-NEXT: rldicl r3, r3, 60, 4
; P8LE-NEXT: sub r5, r6, r5
; P8LE-NEXT: rldic r7, r7, 32, 0
; P8LE-NEXT: mulli r4, r4, 23
; P8LE-NEXT: mulli r3, r3, 23
; P8LE-NEXT: oris r7, r7, 52170
; P8LE-NEXT: ori r7, r7, 12109
; P8LE-NEXT: sub r3, r3, r4
; P8LE-NEXT: sub r3, r4, r3
; P8LE-NEXT: mulhdu r7, r8, r7
; P8LE-NEXT: mtfprd f1, r3
; P8LE-NEXT: li r3, 0

20
llvm/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll
View File

@ -35,12 +35,12 @@ define i64 @test2elt(<2 x i64> %a) local_unnamed_addr #0 {
;
; CHECK-BE-LABEL: test2elt:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: xxswapd vs0, v2
; CHECK-BE-NEXT: xscvuxdsp f1, v2
; CHECK-BE-NEXT: xscvuxdsp f0, f0
; CHECK-BE-NEXT: xscvdpspn v2, f1
; CHECK-BE-NEXT: xscvuxdsp f0, v2
; CHECK-BE-NEXT: xscvdpspn v3, f0
; CHECK-BE-NEXT: vmrgow v2, v2, v3
; CHECK-BE-NEXT: xxswapd vs0, v2
; CHECK-BE-NEXT: xscvuxdsp f0, f0
; CHECK-BE-NEXT: xscvdpspn v2, f0
; CHECK-BE-NEXT: vmrgow v2, v3, v2
; CHECK-BE-NEXT: mfvsrd r3, v2
; CHECK-BE-NEXT: blr
entry:
@ -327,12 +327,12 @@ define i64 @test2elt_signed(<2 x i64> %a) local_unnamed_addr #0 {
;
; CHECK-BE-LABEL: test2elt_signed:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: xxswapd vs0, v2
; CHECK-BE-NEXT: xscvsxdsp f1, v2
; CHECK-BE-NEXT: xscvsxdsp f0, f0
; CHECK-BE-NEXT: xscvdpspn v2, f1
; CHECK-BE-NEXT: xscvsxdsp f0, v2
; CHECK-BE-NEXT: xscvdpspn v3, f0
; CHECK-BE-NEXT: vmrgow v2, v2, v3
; CHECK-BE-NEXT: xxswapd vs0, v2
; CHECK-BE-NEXT: xscvsxdsp f0, f0
; CHECK-BE-NEXT: xscvdpspn v2, f0
; CHECK-BE-NEXT: vmrgow v2, v3, v2
; CHECK-BE-NEXT: mfvsrd r3, v2
; CHECK-BE-NEXT: blr
entry:

10
llvm/test/CodeGen/RISCV/rvv/expandload.ll
View File

@ -1666,20 +1666,20 @@ define <512 x i8> @test_expandload_v512i8_vlen512(ptr %base, <512 x i1> %mask, <
; CHECK-RV32-NEXT: .LBB61_32: # %else114
; CHECK-RV32-NEXT: slli a2, a3, 1
; CHECK-RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; CHECK-RV32-NEXT: vsrl.vx v16, v0, a1
; CHECK-RV32-NEXT: vsrl.vx v24, v0, a1
; CHECK-RV32-NEXT: bgez a2, .LBB61_34
; CHECK-RV32-NEXT: # %bb.33: # %cond.load117
; CHECK-RV32-NEXT: lbu a2, 0(a0)
; CHECK-RV32-NEXT: vmv8r.v v24, v8
; CHECK-RV32-NEXT: vmv8r.v v16, v8
; CHECK-RV32-NEXT: vmv.s.x v9, a2
; CHECK-RV32-NEXT: vsetivli zero, 31, e8, m1, tu, ma
; CHECK-RV32-NEXT: vslideup.vi v8, v9, 30
; CHECK-RV32-NEXT: addi a0, a0, 1
; CHECK-RV32-NEXT: vmv1r.v v24, v8
; CHECK-RV32-NEXT: vmv8r.v v8, v24
; CHECK-RV32-NEXT: vmv1r.v v16, v8
; CHECK-RV32-NEXT: vmv8r.v v8, v16
; CHECK-RV32-NEXT: .LBB61_34: # %else118
; CHECK-RV32-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; CHECK-RV32-NEXT: vmv.x.s a2, v16
; CHECK-RV32-NEXT: vmv.x.s a2, v24
; CHECK-RV32-NEXT: bgez a3, .LBB61_35
; CHECK-RV32-NEXT: j .LBB61_572
; CHECK-RV32-NEXT: .LBB61_35: # %else122

130
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
View File

@ -661,8 +661,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; RV32-NEXT: vmerge.vvm v8, v16, v8, v0
; RV32-NEXT: csrr a7, vlenb
; RV32-NEXT: li t3, 36
; RV32-NEXT: mul a7, a7, t3
; RV32-NEXT: slli a7, a7, 5
; RV32-NEXT: add a7, sp, a7
; RV32-NEXT: addi a7, a7, 16
; RV32-NEXT: vs8r.v v8, (a7) # vscale x 64-byte Folded Spill
@ -682,7 +681,11 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vl8r.v v8, (t1) # vscale x 64-byte Folded Reload
; RV32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
; RV32-NEXT: vmerge.vvm v8, v24, v8, v0
; RV32-NEXT: addi t1, sp, 16
; RV32-NEXT: csrr t1, vlenb
; RV32-NEXT: li t2, 44
; RV32-NEXT: mul t1, t1, t2
; RV32-NEXT: add t1, sp, t1
; RV32-NEXT: addi t1, t1, 16
; RV32-NEXT: vs4r.v v8, (t1) # vscale x 32-byte Folded Spill
; RV32-NEXT: vmv.s.x v0, a7
; RV32-NEXT: addi a3, a3, 12
@ -694,8 +697,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; RV32-NEXT: vmerge.vvm v8, v16, v24, v0
; RV32-NEXT: csrr a7, vlenb
; RV32-NEXT: li t1, 20
; RV32-NEXT: mul a7, a7, t1
; RV32-NEXT: slli a7, a7, 4
; RV32-NEXT: add a7, sp, a7
; RV32-NEXT: addi a7, a7, 16
; RV32-NEXT: vs8r.v v8, (a7) # vscale x 64-byte Folded Spill
@ -733,7 +735,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; RV32-NEXT: vmerge.vvm v8, v8, v16, v0
; RV32-NEXT: csrr a7, vlenb
; RV32-NEXT: li t0, 28
; RV32-NEXT: li t0, 24
; RV32-NEXT: mul a7, a7, t0
; RV32-NEXT: add a7, sp, a7
; RV32-NEXT: addi a7, a7, 16
@ -755,7 +757,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vsetivli zero, 16, e32, m4, ta, ma
; RV32-NEXT: vmerge.vvm v8, v24, v8, v0
; RV32-NEXT: csrr a6, vlenb
; RV32-NEXT: li a7, 44
; RV32-NEXT: li a7, 40
; RV32-NEXT: mul a6, a6, a7
; RV32-NEXT: add a6, sp, a6
; RV32-NEXT: addi a6, a6, 16
@ -772,24 +774,19 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; RV32-NEXT: vmerge.vvm v8, v8, v16, v0
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a4, 12
; RV32-NEXT: mul a1, a1, a4
; RV32-NEXT: slli a1, a1, 3
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; RV32-NEXT: vmv.s.x v0, a3
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a3, 36
; RV32-NEXT: mul a1, a1, a3
; RV32-NEXT: slli a1, a1, 5
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma
; RV32-NEXT: vrgatherei16.vv v24, v8, v6
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: addi a1, sp, 16
; RV32-NEXT: vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a3, 92
@ -812,8 +809,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs4r.v v8, (a1) # vscale x 32-byte Folded Spill
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a3, 20
; RV32-NEXT: mul a1, a1, a3
; RV32-NEXT: slli a1, a1, 4
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
@ -835,12 +831,6 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; RV32-NEXT: vmerge.vvm v8, v8, v16, v0
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 84
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 72
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
@ -860,30 +850,36 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs4r.v v28, (a1) # vscale x 32-byte Folded Spill
; RV32-NEXT: addi a1, sp, 16
; RV32-NEXT: vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 60
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl4r.v v20, (a1) # vscale x 32-byte Folded Reload
; RV32-NEXT: vmv.v.v v20, v16
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 60
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs4r.v v20, (a1) # vscale x 32-byte Folded Spill
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 44
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl4r.v v16, (a1) # vscale x 32-byte Folded Reload
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; RV32-NEXT: vmv.v.v v16, v8
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 60
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs4r.v v16, (a1) # vscale x 32-byte Folded Spill
; RV32-NEXT: addi a1, sp, 16
; RV32-NEXT: vl4r.v v8, (a1) # vscale x 32-byte Folded Reload
; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma
; RV32-NEXT: vrgatherei16.vv v28, v8, v3
; RV32-NEXT: vrgatherei16.vv v20, v16, v3
; RV32-NEXT: vsetivli zero, 10, e32, m4, tu, ma
; RV32-NEXT: vmv.v.v v28, v24
; RV32-NEXT: vmv.v.v v20, v24
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 6
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs4r.v v20, (a1) # vscale x 32-byte Folded Spill
; RV32-NEXT: lui a1, %hi(.LCPI27_4)
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_4)
; RV32-NEXT: lui a2, %hi(.LCPI27_5)
@ -891,13 +887,25 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vsetivli zero, 16, e16, m2, ta, ma
; RV32-NEXT: vle16.v v24, (a2)
; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; RV32-NEXT: vle16.v v8, (a1)
; RV32-NEXT: vle16.v v16, (a1)
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 84
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs1r.v v16, (a1) # vscale x 8-byte Folded Spill
; RV32-NEXT: lui a1, %hi(.LCPI27_7)
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_7)
; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma
; RV32-NEXT: vle16.v v10, (a1)
; RV32-NEXT: vle16.v v16, (a1)
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 28
; RV32-NEXT: li a2, 76
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vs2r.v v16, (a1) # vscale x 16-byte Folded Spill
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 24
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
@ -909,18 +917,29 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl4r.v v20, (a1) # vscale x 32-byte Folded Reload
; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma
; RV32-NEXT: vrgatherei16.vv v24, v20, v8
; RV32-NEXT: vsetivli zero, 10, e32, m4, tu, ma
; RV32-NEXT: vmv.v.v v24, v16
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 12
; RV32-NEXT: li a2, 84
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl1r.v v7, (a1) # vscale x 8-byte Folded Reload
; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma
; RV32-NEXT: vrgatherei16.vv v24, v20, v7
; RV32-NEXT: vsetivli zero, 10, e32, m4, tu, ma
; RV32-NEXT: vmv.v.v v24, v16
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: slli a1, a1, 3
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 76
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl2r.v v28, (a1) # vscale x 16-byte Folded Reload
; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma
; RV32-NEXT: vrgatherei16.vv v16, v0, v10
; RV32-NEXT: vrgatherei16.vv v16, v0, v28
; RV32-NEXT: lui a1, %hi(.LCPI27_6)
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_6)
; RV32-NEXT: lui a2, %hi(.LCPI27_8)
@ -934,7 +953,7 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, ma
; RV32-NEXT: vle16.v v5, (a2)
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 44
; RV32-NEXT: li a2, 40
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
@ -942,12 +961,6 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: vrgatherei16.vv v0, v20, v4
; RV32-NEXT: vsetivli zero, 10, e32, m4, tu, ma
; RV32-NEXT: vmv.v.v v0, v16
; RV32-NEXT: csrr a1, vlenb
; RV32-NEXT: li a2, 84
; RV32-NEXT: mul a1, a1, a2
; RV32-NEXT: add a1, sp, a1
; RV32-NEXT: addi a1, a1, 16
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma
; RV32-NEXT: vrgatherei16.vv v16, v8, v6
; RV32-NEXT: csrr a1, vlenb
@ -968,7 +981,12 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
; RV32-NEXT: addi a1, a0, 192
; RV32-NEXT: vse32.v v24, (a1)
; RV32-NEXT: addi a1, a0, 128
; RV32-NEXT: vse32.v v28, (a1)
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: slli a2, a2, 6
; RV32-NEXT: add a2, sp, a2
; RV32-NEXT: addi a2, a2, 16
; RV32-NEXT: vl4r.v v8, (a2) # vscale x 32-byte Folded Reload
; RV32-NEXT: vse32.v v8, (a1)
; RV32-NEXT: addi a1, a0, 64
; RV32-NEXT: csrr a2, vlenb
; RV32-NEXT: li a3, 60

72
llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
View File

@ -416,14 +416,14 @@ define <vscale x 32 x i1> @reverse_nxv32i1(<vscale x 32 x i1> %a) {
; RV32-BITS-UNKNOWN-NEXT: vsetvli a1, zero, e16, m2, ta, ma
; RV32-BITS-UNKNOWN-NEXT: vrsub.vx v16, v12, a0
; RV32-BITS-UNKNOWN-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV32-BITS-UNKNOWN-NEXT: vmerge.vim v12, v8, 1, v0
; RV32-BITS-UNKNOWN-NEXT: vmerge.vim v8, v8, 1, v0
; RV32-BITS-UNKNOWN-NEXT: vsetvli a0, zero, e8, m1, ta, ma
; RV32-BITS-UNKNOWN-NEXT: vrgatherei16.vv v11, v12, v16
; RV32-BITS-UNKNOWN-NEXT: vrgatherei16.vv v10, v13, v16
; RV32-BITS-UNKNOWN-NEXT: vrgatherei16.vv v9, v14, v16
; RV32-BITS-UNKNOWN-NEXT: vrgatherei16.vv v8, v15, v16
; RV32-BITS-UNKNOWN-NEXT: vrgatherei16.vv v15, v8, v16
; RV32-BITS-UNKNOWN-NEXT: vrgatherei16.vv v14, v9, v16
; RV32-BITS-UNKNOWN-NEXT: vrgatherei16.vv v13, v10, v16
; RV32-BITS-UNKNOWN-NEXT: vrgatherei16.vv v12, v11, v16
; RV32-BITS-UNKNOWN-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV32-BITS-UNKNOWN-NEXT: vmsne.vi v0, v8, 0
; RV32-BITS-UNKNOWN-NEXT: vmsne.vi v0, v12, 0
; RV32-BITS-UNKNOWN-NEXT: ret
;
; RV32-BITS-256-LABEL: reverse_nxv32i1:
@ -437,14 +437,14 @@ define <vscale x 32 x i1> @reverse_nxv32i1(<vscale x 32 x i1> %a) {
; RV32-BITS-256-NEXT: vsetvli a1, zero, e8, m1, ta, ma
; RV32-BITS-256-NEXT: vrsub.vx v16, v12, a0
; RV32-BITS-256-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV32-BITS-256-NEXT: vmerge.vim v12, v8, 1, v0
; RV32-BITS-256-NEXT: vmerge.vim v8, v8, 1, v0
; RV32-BITS-256-NEXT: vsetvli a0, zero, e8, m1, ta, ma
; RV32-BITS-256-NEXT: vrgather.vv v11, v12, v16
; RV32-BITS-256-NEXT: vrgather.vv v10, v13, v16
; RV32-BITS-256-NEXT: vrgather.vv v9, v14, v16
; RV32-BITS-256-NEXT: vrgather.vv v8, v15, v16
; RV32-BITS-256-NEXT: vrgather.vv v15, v8, v16
; RV32-BITS-256-NEXT: vrgather.vv v14, v9, v16
; RV32-BITS-256-NEXT: vrgather.vv v13, v10, v16
; RV32-BITS-256-NEXT: vrgather.vv v12, v11, v16
; RV32-BITS-256-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV32-BITS-256-NEXT: vmsne.vi v0, v8, 0
; RV32-BITS-256-NEXT: vmsne.vi v0, v12, 0
; RV32-BITS-256-NEXT: ret
;
; RV32-BITS-512-LABEL: reverse_nxv32i1:
@ -458,14 +458,14 @@ define <vscale x 32 x i1> @reverse_nxv32i1(<vscale x 32 x i1> %a) {
; RV32-BITS-512-NEXT: vsetvli a1, zero, e8, m1, ta, ma
; RV32-BITS-512-NEXT: vrsub.vx v16, v12, a0
; RV32-BITS-512-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV32-BITS-512-NEXT: vmerge.vim v12, v8, 1, v0
; RV32-BITS-512-NEXT: vmerge.vim v8, v8, 1, v0
; RV32-BITS-512-NEXT: vsetvli a0, zero, e8, m1, ta, ma
; RV32-BITS-512-NEXT: vrgather.vv v11, v12, v16
; RV32-BITS-512-NEXT: vrgather.vv v10, v13, v16
; RV32-BITS-512-NEXT: vrgather.vv v9, v14, v16
; RV32-BITS-512-NEXT: vrgather.vv v8, v15, v16
; RV32-BITS-512-NEXT: vrgather.vv v15, v8, v16
; RV32-BITS-512-NEXT: vrgather.vv v14, v9, v16
; RV32-BITS-512-NEXT: vrgather.vv v13, v10, v16
; RV32-BITS-512-NEXT: vrgather.vv v12, v11, v16
; RV32-BITS-512-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV32-BITS-512-NEXT: vmsne.vi v0, v8, 0
; RV32-BITS-512-NEXT: vmsne.vi v0, v12, 0
; RV32-BITS-512-NEXT: ret
;
; RV64-BITS-UNKNOWN-LABEL: reverse_nxv32i1:
@ -479,14 +479,14 @@ define <vscale x 32 x i1> @reverse_nxv32i1(<vscale x 32 x i1> %a) {
; RV64-BITS-UNKNOWN-NEXT: vsetvli a1, zero, e16, m2, ta, ma
; RV64-BITS-UNKNOWN-NEXT: vrsub.vx v16, v12, a0
; RV64-BITS-UNKNOWN-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV64-BITS-UNKNOWN-NEXT: vmerge.vim v12, v8, 1, v0
; RV64-BITS-UNKNOWN-NEXT: vmerge.vim v8, v8, 1, v0
; RV64-BITS-UNKNOWN-NEXT: vsetvli a0, zero, e8, m1, ta, ma
; RV64-BITS-UNKNOWN-NEXT: vrgatherei16.vv v11, v12, v16
; RV64-BITS-UNKNOWN-NEXT: vrgatherei16.vv v10, v13, v16
; RV64-BITS-UNKNOWN-NEXT: vrgatherei16.vv v9, v14, v16
; RV64-BITS-UNKNOWN-NEXT: vrgatherei16.vv v8, v15, v16
; RV64-BITS-UNKNOWN-NEXT: vrgatherei16.vv v15, v8, v16
; RV64-BITS-UNKNOWN-NEXT: vrgatherei16.vv v14, v9, v16
; RV64-BITS-UNKNOWN-NEXT: vrgatherei16.vv v13, v10, v16
; RV64-BITS-UNKNOWN-NEXT: vrgatherei16.vv v12, v11, v16
; RV64-BITS-UNKNOWN-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV64-BITS-UNKNOWN-NEXT: vmsne.vi v0, v8, 0
; RV64-BITS-UNKNOWN-NEXT: vmsne.vi v0, v12, 0
; RV64-BITS-UNKNOWN-NEXT: ret
;
; RV64-BITS-256-LABEL: reverse_nxv32i1:
@ -500,14 +500,14 @@ define <vscale x 32 x i1> @reverse_nxv32i1(<vscale x 32 x i1> %a) {
; RV64-BITS-256-NEXT: vsetvli a1, zero, e8, m1, ta, ma
; RV64-BITS-256-NEXT: vrsub.vx v16, v12, a0
; RV64-BITS-256-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV64-BITS-256-NEXT: vmerge.vim v12, v8, 1, v0
; RV64-BITS-256-NEXT: vmerge.vim v8, v8, 1, v0
; RV64-BITS-256-NEXT: vsetvli a0, zero, e8, m1, ta, ma
; RV64-BITS-256-NEXT: vrgather.vv v11, v12, v16
; RV64-BITS-256-NEXT: vrgather.vv v10, v13, v16
; RV64-BITS-256-NEXT: vrgather.vv v9, v14, v16
; RV64-BITS-256-NEXT: vrgather.vv v8, v15, v16
; RV64-BITS-256-NEXT: vrgather.vv v15, v8, v16
; RV64-BITS-256-NEXT: vrgather.vv v14, v9, v16
; RV64-BITS-256-NEXT: vrgather.vv v13, v10, v16
; RV64-BITS-256-NEXT: vrgather.vv v12, v11, v16
; RV64-BITS-256-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV64-BITS-256-NEXT: vmsne.vi v0, v8, 0
; RV64-BITS-256-NEXT: vmsne.vi v0, v12, 0
; RV64-BITS-256-NEXT: ret
;
; RV64-BITS-512-LABEL: reverse_nxv32i1:
@ -521,14 +521,14 @@ define <vscale x 32 x i1> @reverse_nxv32i1(<vscale x 32 x i1> %a) {
; RV64-BITS-512-NEXT: vsetvli a1, zero, e8, m1, ta, ma
; RV64-BITS-512-NEXT: vrsub.vx v16, v12, a0
; RV64-BITS-512-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV64-BITS-512-NEXT: vmerge.vim v12, v8, 1, v0
; RV64-BITS-512-NEXT: vmerge.vim v8, v8, 1, v0
; RV64-BITS-512-NEXT: vsetvli a0, zero, e8, m1, ta, ma
; RV64-BITS-512-NEXT: vrgather.vv v11, v12, v16
; RV64-BITS-512-NEXT: vrgather.vv v10, v13, v16
; RV64-BITS-512-NEXT: vrgather.vv v9, v14, v16
; RV64-BITS-512-NEXT: vrgather.vv v8, v15, v16
; RV64-BITS-512-NEXT: vrgather.vv v15, v8, v16
; RV64-BITS-512-NEXT: vrgather.vv v14, v9, v16
; RV64-BITS-512-NEXT: vrgather.vv v13, v10, v16
; RV64-BITS-512-NEXT: vrgather.vv v12, v11, v16
; RV64-BITS-512-NEXT: vsetvli a0, zero, e8, m4, ta, ma
; RV64-BITS-512-NEXT: vmsne.vi v0, v8, 0
; RV64-BITS-512-NEXT: vmsne.vi v0, v12, 0
; RV64-BITS-512-NEXT: ret
%res = call <vscale x 32 x i1> @llvm.vector.reverse.nxv32i1(<vscale x 32 x i1> %a)
ret <vscale x 32 x i1> %res

80
llvm/test/CodeGen/RISCV/rvv/nontemporal-vp-scalable.ll
View File

@ -37772,18 +37772,18 @@ define void @test_nontemporal_vp_scatter_nxv64i8_P1(<vscale x 64 x i8> %val, <vs
; CHECK-RV32VC-LABEL: test_nontemporal_vp_scatter_nxv64i8_P1:
; CHECK-RV32VC: # %bb.0:
; CHECK-RV32VC-NEXT: csrr a1, vlenb
; CHECK-RV32VC-NEXT: slli a5, a1, 4
; CHECK-RV32VC-NEXT: slli a6, a1, 4
; CHECK-RV32VC-NEXT: slli a2, a1, 2
; CHECK-RV32VC-NEXT: slli a6, a1, 3
; CHECK-RV32VC-NEXT: slli a5, a1, 3
; CHECK-RV32VC-NEXT: mv a4, a3
; CHECK-RV32VC-NEXT: bltu a3, a2, .LBB915_2
; CHECK-RV32VC-NEXT: # %bb.1:
; CHECK-RV32VC-NEXT: mv a4, a2
; CHECK-RV32VC-NEXT: .LBB915_2:
; CHECK-RV32VC-NEXT: vl8re32.v v0, (a0)
; CHECK-RV32VC-NEXT: add a7, a0, a5
; CHECK-RV32VC-NEXT: add a6, a6, a0
; CHECK-RV32VC-NEXT: slli a1, a1, 1
; CHECK-RV32VC-NEXT: add a0, a0, a6
; CHECK-RV32VC-NEXT: add a0, a0, a5
; CHECK-RV32VC-NEXT: mv a5, a4
; CHECK-RV32VC-NEXT: bltu a4, a1, .LBB915_4
; CHECK-RV32VC-NEXT: # %bb.3:
@ -37791,11 +37791,11 @@ define void @test_nontemporal_vp_scatter_nxv64i8_P1(<vscale x 64 x i8> %val, <vs
; CHECK-RV32VC-NEXT: .LBB915_4:
; CHECK-RV32VC-NEXT: addi sp, sp, -16
; CHECK-RV32VC-NEXT: .cfi_def_cfa_offset 16
; CHECK-RV32VC-NEXT: csrr a6, vlenb
; CHECK-RV32VC-NEXT: slli a6, a6, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a6
; CHECK-RV32VC-NEXT: csrr a7, vlenb
; CHECK-RV32VC-NEXT: slli a7, a7, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a7
; CHECK-RV32VC-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a7)
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a6)
; CHECK-RV32VC-NEXT: addi a6, sp, 16
; CHECK-RV32VC-NEXT: vs8r.v v24, (a6) # vscale x 64-byte Folded Spill
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a0)
@ -38397,18 +38397,18 @@ define void @test_nontemporal_vp_scatter_nxv64i8_PALL(<vscale x 64 x i8> %val, <
; CHECK-RV32VC-LABEL: test_nontemporal_vp_scatter_nxv64i8_PALL:
; CHECK-RV32VC: # %bb.0:
; CHECK-RV32VC-NEXT: csrr a1, vlenb
; CHECK-RV32VC-NEXT: slli a5, a1, 4
; CHECK-RV32VC-NEXT: slli a6, a1, 4
; CHECK-RV32VC-NEXT: slli a2, a1, 2
; CHECK-RV32VC-NEXT: slli a6, a1, 3
; CHECK-RV32VC-NEXT: slli a5, a1, 3
; CHECK-RV32VC-NEXT: mv a4, a3
; CHECK-RV32VC-NEXT: bltu a3, a2, .LBB916_2
; CHECK-RV32VC-NEXT: # %bb.1:
; CHECK-RV32VC-NEXT: mv a4, a2
; CHECK-RV32VC-NEXT: .LBB916_2:
; CHECK-RV32VC-NEXT: vl8re32.v v0, (a0)
; CHECK-RV32VC-NEXT: add a7, a0, a5
; CHECK-RV32VC-NEXT: add a6, a6, a0
; CHECK-RV32VC-NEXT: slli a1, a1, 1
; CHECK-RV32VC-NEXT: add a0, a0, a6
; CHECK-RV32VC-NEXT: add a0, a0, a5
; CHECK-RV32VC-NEXT: mv a5, a4
; CHECK-RV32VC-NEXT: bltu a4, a1, .LBB916_4
; CHECK-RV32VC-NEXT: # %bb.3:
@ -38416,11 +38416,11 @@ define void @test_nontemporal_vp_scatter_nxv64i8_PALL(<vscale x 64 x i8> %val, <
; CHECK-RV32VC-NEXT: .LBB916_4:
; CHECK-RV32VC-NEXT: addi sp, sp, -16
; CHECK-RV32VC-NEXT: .cfi_def_cfa_offset 16
; CHECK-RV32VC-NEXT: csrr a6, vlenb
; CHECK-RV32VC-NEXT: slli a6, a6, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a6
; CHECK-RV32VC-NEXT: csrr a7, vlenb
; CHECK-RV32VC-NEXT: slli a7, a7, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a7
; CHECK-RV32VC-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a7)
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a6)
; CHECK-RV32VC-NEXT: addi a6, sp, 16
; CHECK-RV32VC-NEXT: vs8r.v v24, (a6) # vscale x 64-byte Folded Spill
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a0)
@ -39022,18 +39022,18 @@ define void @test_nontemporal_vp_scatter_nxv64i8_S1(<vscale x 64 x i8> %val, <vs
; CHECK-RV32VC-LABEL: test_nontemporal_vp_scatter_nxv64i8_S1:
; CHECK-RV32VC: # %bb.0:
; CHECK-RV32VC-NEXT: csrr a1, vlenb
; CHECK-RV32VC-NEXT: slli a5, a1, 4
; CHECK-RV32VC-NEXT: slli a6, a1, 4
; CHECK-RV32VC-NEXT: slli a2, a1, 2
; CHECK-RV32VC-NEXT: slli a6, a1, 3
; CHECK-RV32VC-NEXT: slli a5, a1, 3
; CHECK-RV32VC-NEXT: mv a4, a3
; CHECK-RV32VC-NEXT: bltu a3, a2, .LBB917_2
; CHECK-RV32VC-NEXT: # %bb.1:
; CHECK-RV32VC-NEXT: mv a4, a2
; CHECK-RV32VC-NEXT: .LBB917_2:
; CHECK-RV32VC-NEXT: vl8re32.v v0, (a0)
; CHECK-RV32VC-NEXT: add a7, a0, a5
; CHECK-RV32VC-NEXT: add a6, a6, a0
; CHECK-RV32VC-NEXT: slli a1, a1, 1
; CHECK-RV32VC-NEXT: add a0, a0, a6
; CHECK-RV32VC-NEXT: add a0, a0, a5
; CHECK-RV32VC-NEXT: mv a5, a4
; CHECK-RV32VC-NEXT: bltu a4, a1, .LBB917_4
; CHECK-RV32VC-NEXT: # %bb.3:
@ -39041,11 +39041,11 @@ define void @test_nontemporal_vp_scatter_nxv64i8_S1(<vscale x 64 x i8> %val, <vs
; CHECK-RV32VC-NEXT: .LBB917_4:
; CHECK-RV32VC-NEXT: addi sp, sp, -16
; CHECK-RV32VC-NEXT: .cfi_def_cfa_offset 16
; CHECK-RV32VC-NEXT: csrr a6, vlenb
; CHECK-RV32VC-NEXT: slli a6, a6, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a6
; CHECK-RV32VC-NEXT: csrr a7, vlenb
; CHECK-RV32VC-NEXT: slli a7, a7, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a7
; CHECK-RV32VC-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a7)
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a6)
; CHECK-RV32VC-NEXT: addi a6, sp, 16
; CHECK-RV32VC-NEXT: vs8r.v v24, (a6) # vscale x 64-byte Folded Spill
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a0)
@ -39647,18 +39647,18 @@ define void @test_nontemporal_vp_scatter_nxv64i8_ALL(<vscale x 64 x i8> %val, <v
; CHECK-RV32VC-LABEL: test_nontemporal_vp_scatter_nxv64i8_ALL:
; CHECK-RV32VC: # %bb.0:
; CHECK-RV32VC-NEXT: csrr a1, vlenb
; CHECK-RV32VC-NEXT: slli a5, a1, 4
; CHECK-RV32VC-NEXT: slli a6, a1, 4
; CHECK-RV32VC-NEXT: slli a2, a1, 2
; CHECK-RV32VC-NEXT: slli a6, a1, 3
; CHECK-RV32VC-NEXT: slli a5, a1, 3
; CHECK-RV32VC-NEXT: mv a4, a3
; CHECK-RV32VC-NEXT: bltu a3, a2, .LBB918_2
; CHECK-RV32VC-NEXT: # %bb.1:
; CHECK-RV32VC-NEXT: mv a4, a2
; CHECK-RV32VC-NEXT: .LBB918_2:
; CHECK-RV32VC-NEXT: vl8re32.v v0, (a0)
; CHECK-RV32VC-NEXT: add a7, a0, a5
; CHECK-RV32VC-NEXT: add a6, a6, a0
; CHECK-RV32VC-NEXT: slli a1, a1, 1
; CHECK-RV32VC-NEXT: add a0, a0, a6
; CHECK-RV32VC-NEXT: add a0, a0, a5
; CHECK-RV32VC-NEXT: mv a5, a4
; CHECK-RV32VC-NEXT: bltu a4, a1, .LBB918_4
; CHECK-RV32VC-NEXT: # %bb.3:
@ -39666,11 +39666,11 @@ define void @test_nontemporal_vp_scatter_nxv64i8_ALL(<vscale x 64 x i8> %val, <v
; CHECK-RV32VC-NEXT: .LBB918_4:
; CHECK-RV32VC-NEXT: addi sp, sp, -16
; CHECK-RV32VC-NEXT: .cfi_def_cfa_offset 16
; CHECK-RV32VC-NEXT: csrr a6, vlenb
; CHECK-RV32VC-NEXT: slli a6, a6, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a6
; CHECK-RV32VC-NEXT: csrr a7, vlenb
; CHECK-RV32VC-NEXT: slli a7, a7, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a7
; CHECK-RV32VC-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a7)
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a6)
; CHECK-RV32VC-NEXT: addi a6, sp, 16
; CHECK-RV32VC-NEXT: vs8r.v v24, (a6) # vscale x 64-byte Folded Spill
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a0)
@ -40271,18 +40271,18 @@ define void @test_nontemporal_vp_scatter_nxv64i8_DEFAULT(<vscale x 64 x i8> %val
; CHECK-RV32VC-LABEL: test_nontemporal_vp_scatter_nxv64i8_DEFAULT:
; CHECK-RV32VC: # %bb.0:
; CHECK-RV32VC-NEXT: csrr a1, vlenb
; CHECK-RV32VC-NEXT: slli a5, a1, 4
; CHECK-RV32VC-NEXT: slli a6, a1, 4
; CHECK-RV32VC-NEXT: slli a2, a1, 2
; CHECK-RV32VC-NEXT: slli a6, a1, 3
; CHECK-RV32VC-NEXT: slli a5, a1, 3
; CHECK-RV32VC-NEXT: mv a4, a3
; CHECK-RV32VC-NEXT: bltu a3, a2, .LBB919_2
; CHECK-RV32VC-NEXT: # %bb.1:
; CHECK-RV32VC-NEXT: mv a4, a2
; CHECK-RV32VC-NEXT: .LBB919_2:
; CHECK-RV32VC-NEXT: vl8re32.v v0, (a0)
; CHECK-RV32VC-NEXT: add a7, a0, a5
; CHECK-RV32VC-NEXT: add a6, a6, a0
; CHECK-RV32VC-NEXT: slli a1, a1, 1
; CHECK-RV32VC-NEXT: add a0, a0, a6
; CHECK-RV32VC-NEXT: add a0, a0, a5
; CHECK-RV32VC-NEXT: mv a5, a4
; CHECK-RV32VC-NEXT: bltu a4, a1, .LBB919_4
; CHECK-RV32VC-NEXT: # %bb.3:
@ -40290,11 +40290,11 @@ define void @test_nontemporal_vp_scatter_nxv64i8_DEFAULT(<vscale x 64 x i8> %val
; CHECK-RV32VC-NEXT: .LBB919_4:
; CHECK-RV32VC-NEXT: addi sp, sp, -16
; CHECK-RV32VC-NEXT: .cfi_def_cfa_offset 16
; CHECK-RV32VC-NEXT: csrr a6, vlenb
; CHECK-RV32VC-NEXT: slli a6, a6, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a6
; CHECK-RV32VC-NEXT: csrr a7, vlenb
; CHECK-RV32VC-NEXT: slli a7, a7, 3
; CHECK-RV32VC-NEXT: sub sp, sp, a7
; CHECK-RV32VC-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a7)
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a6)
; CHECK-RV32VC-NEXT: addi a6, sp, 16
; CHECK-RV32VC-NEXT: vs8r.v v24, (a6) # vscale x 64-byte Folded Spill
; CHECK-RV32VC-NEXT: vl8re32.v v24, (a0)

41
llvm/test/CodeGen/Thumb2/mve-soft-float-abi.ll
View File

@ -348,38 +348,35 @@ entry:
define <4 x float> @vector_add_f32(<4 x float> %lhs, <4 x float> %rhs) {
; CHECK-MVE-LABEL: vector_add_f32:
; CHECK-MVE: @ %bb.0: @ %entry
; CHECK-MVE-NEXT: .save {r4, r5, r6, r7, lr}
; CHECK-MVE-NEXT: push {r4, r5, r6, r7, lr}
; CHECK-MVE-NEXT: .pad #4
; CHECK-MVE-NEXT: sub sp, #4
; CHECK-MVE-NEXT: .save {r4, r5, r6, r7, r8, lr}
; CHECK-MVE-NEXT: push.w {r4, r5, r6, r7, r8, lr}
; CHECK-MVE-NEXT: .vsave {d8, d9}
; CHECK-MVE-NEXT: vpush {d8, d9}
; CHECK-MVE-NEXT: mov r4, r0
; CHECK-MVE-NEXT: mov r8, r0
; CHECK-MVE-NEXT: add r0, sp, #40
; CHECK-MVE-NEXT: vldrw.u32 q4, [r0]
; CHECK-MVE-NEXT: mov r6, r1
; CHECK-MVE-NEXT: mov r7, r1
; CHECK-MVE-NEXT: mov r0, r3
; CHECK-MVE-NEXT: mov r5, r2
; CHECK-MVE-NEXT: vmov r7, r1, d9
; CHECK-MVE-NEXT: mov r6, r2
; CHECK-MVE-NEXT: vmov r4, r1, d9
; CHECK-MVE-NEXT: bl __aeabi_fadd
; CHECK-MVE-NEXT: vmov s19, r0
; CHECK-MVE-NEXT: mov r0, r5
; CHECK-MVE-NEXT: mov r1, r7
; CHECK-MVE-NEXT: bl __aeabi_fadd
; CHECK-MVE-NEXT: vmov r5, r1, d8
; CHECK-MVE-NEXT: vmov s18, r0
; CHECK-MVE-NEXT: mov r5, r0
; CHECK-MVE-NEXT: mov r0, r6
; CHECK-MVE-NEXT: mov r1, r4
; CHECK-MVE-NEXT: bl __aeabi_fadd
; CHECK-MVE-NEXT: vmov s17, r0
; CHECK-MVE-NEXT: mov r0, r4
; CHECK-MVE-NEXT: mov r1, r5
; CHECK-MVE-NEXT: vmov r6, r1, d8
; CHECK-MVE-NEXT: mov r4, r0
; CHECK-MVE-NEXT: mov r0, r7
; CHECK-MVE-NEXT: bl __aeabi_fadd
; CHECK-MVE-NEXT: vmov s16, r0
; CHECK-MVE-NEXT: vmov r2, r3, d9
; CHECK-MVE-NEXT: vmov r0, r1, d8
; CHECK-MVE-NEXT: mov r7, r0
; CHECK-MVE-NEXT: mov r0, r8
; CHECK-MVE-NEXT: mov r1, r6
; CHECK-MVE-NEXT: bl __aeabi_fadd
; CHECK-MVE-NEXT: mov r1, r7
; CHECK-MVE-NEXT: mov r2, r4
; CHECK-MVE-NEXT: mov r3, r5
; CHECK-MVE-NEXT: vpop {d8, d9}
; CHECK-MVE-NEXT: add sp, #4
; CHECK-MVE-NEXT: pop {r4, r5, r6, r7, pc}
; CHECK-MVE-NEXT: pop.w {r4, r5, r6, r7, r8, pc}
;
; CHECK-BE-LABEL: vector_add_f32:
; CHECK-BE: @ %bb.0: @ %entry

414
llvm/test/CodeGen/Thumb2/mve-vld3.ll
View File

@ -33,53 +33,29 @@ entry:
}
define void @vld3_v4i32(ptr %src, ptr %dst) {
; CHECK-LV-LABEL: vld3_v4i32:
; CHECK-LV: @ %bb.0: @ %entry
; CHECK-LV-NEXT: .vsave {d8, d9}
; CHECK-LV-NEXT: vpush {d8, d9}
; CHECK-LV-NEXT: vldrw.u32 q0, [r0, #16]
; CHECK-LV-NEXT: vldrw.u32 q1, [r0]
; CHECK-LV-NEXT: vldrw.u32 q4, [r0, #32]
; CHECK-LV-NEXT: vmov.f32 s10, s2
; CHECK-LV-NEXT: vmov.f32 s13, s0
; CHECK-LV-NEXT: vmov.f32 s14, s3
; CHECK-LV-NEXT: vmov.f32 s8, s4
; CHECK-LV-NEXT: vmov.f32 s9, s7
; CHECK-LV-NEXT: vmov.f32 s12, s5
; CHECK-LV-NEXT: vmov.f32 s15, s18
; CHECK-LV-NEXT: vmov.f32 s11, s17
; CHECK-LV-NEXT: vadd.i32 q2, q2, q3
; CHECK-LV-NEXT: vmov.f32 s0, s6
; CHECK-LV-NEXT: vmov.f32 s2, s16
; CHECK-LV-NEXT: vmov.f32 s3, s19
; CHECK-LV-NEXT: vadd.i32 q0, q2, q0
; CHECK-LV-NEXT: vstrw.32 q0, [r1]
; CHECK-LV-NEXT: vpop {d8, d9}
; CHECK-LV-NEXT: bx lr
;
; CHECK-LIS-LABEL: vld3_v4i32:
; CHECK-LIS: @ %bb.0: @ %entry
; CHECK-LIS-NEXT: .vsave {d8, d9}
; CHECK-LIS-NEXT: vpush {d8, d9}
; CHECK-LIS-NEXT: vldrw.u32 q0, [r0, #16]
; CHECK-LIS-NEXT: vldrw.u32 q1, [r0]
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #32]
; CHECK-LIS-NEXT: vmov.f32 s10, s2
; CHECK-LIS-NEXT: vmov.f32 s17, s0
; CHECK-LIS-NEXT: vmov.f32 s18, s3
; CHECK-LIS-NEXT: vmov.f32 s8, s4
; CHECK-LIS-NEXT: vmov.f32 s9, s7
; CHECK-LIS-NEXT: vmov.f32 s16, s5
; CHECK-LIS-NEXT: vmov.f32 s19, s14
; CHECK-LIS-NEXT: vmov.f32 s11, s13
; CHECK-LIS-NEXT: vadd.i32 q2, q2, q4
; CHECK-LIS-NEXT: vmov.f32 s0, s6
; CHECK-LIS-NEXT: vmov.f32 s2, s12
; CHECK-LIS-NEXT: vmov.f32 s3, s15
; CHECK-LIS-NEXT: vadd.i32 q0, q2, q0
; CHECK-LIS-NEXT: vstrw.32 q0, [r1]
; CHECK-LIS-NEXT: vpop {d8, d9}
; CHECK-LIS-NEXT: bx lr
; CHECK-LABEL: vld3_v4i32:
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .vsave {d8, d9}
; CHECK-NEXT: vpush {d8, d9}
; CHECK-NEXT: vldrw.u32 q0, [r0, #16]
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: vldrw.u32 q4, [r0, #32]
; CHECK-NEXT: vmov.f32 s10, s2
; CHECK-NEXT: vmov.f32 s13, s0
; CHECK-NEXT: vmov.f32 s14, s3
; CHECK-NEXT: vmov.f32 s8, s4
; CHECK-NEXT: vmov.f32 s9, s7
; CHECK-NEXT: vmov.f32 s12, s5
; CHECK-NEXT: vmov.f32 s15, s18
; CHECK-NEXT: vmov.f32 s11, s17
; CHECK-NEXT: vadd.i32 q2, q2, q3
; CHECK-NEXT: vmov.f32 s0, s6
; CHECK-NEXT: vmov.f32 s2, s16
; CHECK-NEXT: vmov.f32 s3, s19
; CHECK-NEXT: vadd.i32 q0, q2, q0
; CHECK-NEXT: vstrw.32 q0, [r1]
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: bx lr
entry:
%l1 = load <12 x i32>, ptr %src, align 4
@ -93,87 +69,46 @@ entry:
}
define void @vld3_v8i32(ptr %src, ptr %dst) {
; CHECK-LV-LABEL: vld3_v8i32:
; CHECK-LV: @ %bb.0: @ %entry
; CHECK-LV-NEXT: .vsave {d8, d9, d10, d11}
; CHECK-LV-NEXT: vpush {d8, d9, d10, d11}
; CHECK-LV-NEXT: vldrw.u32 q0, [r0, #64]
; CHECK-LV-NEXT: vldrw.u32 q1, [r0, #48]
; CHECK-LV-NEXT: vldrw.u32 q4, [r0, #80]
; CHECK-LV-NEXT: vmov.f32 s10, s2
; CHECK-LV-NEXT: vmov.f32 s13, s0
; CHECK-LV-NEXT: vmov.f32 s14, s3
; CHECK-LV-NEXT: vmov.f32 s8, s4
; CHECK-LV-NEXT: vmov.f32 s9, s7
; CHECK-LV-NEXT: vmov.f32 s12, s5
; CHECK-LV-NEXT: vmov.f32 s15, s18
; CHECK-LV-NEXT: vmov.f32 s11, s17
; CHECK-LV-NEXT: vadd.i32 q2, q2, q3
; CHECK-LV-NEXT: vmov.f32 s0, s6
; CHECK-LV-NEXT: vmov.f32 s2, s16
; CHECK-LV-NEXT: vldrw.u32 q1, [r0, #16]
; CHECK-LV-NEXT: vmov.f32 s3, s19
; CHECK-LV-NEXT: vldrw.u32 q3, [r0, #32]
; CHECK-LV-NEXT: vadd.i32 q0, q2, q0
; CHECK-LV-NEXT: vldrw.u32 q2, [r0]
; CHECK-LV-NEXT: vmov.f32 s17, s4
; CHECK-LV-NEXT: vstrw.32 q0, [r1, #16]
; CHECK-LV-NEXT: vmov.f32 s18, s7
; CHECK-LV-NEXT: vmov.f32 s22, s6
; CHECK-LV-NEXT: vmov.f32 s16, s9
; CHECK-LV-NEXT: vmov.f32 s19, s14
; CHECK-LV-NEXT: vmov.f32 s20, s8
; CHECK-LV-NEXT: vmov.f32 s21, s11
; CHECK-LV-NEXT: vmov.f32 s23, s13
; CHECK-LV-NEXT: vadd.i32 q4, q5, q4
; CHECK-LV-NEXT: vmov.f32 s4, s10
; CHECK-LV-NEXT: vmov.f32 s6, s12
; CHECK-LV-NEXT: vmov.f32 s7, s15
; CHECK-LV-NEXT: vadd.i32 q1, q4, q1
; CHECK-LV-NEXT: vstrw.32 q1, [r1]
; CHECK-LV-NEXT: vpop {d8, d9, d10, d11}
; CHECK-LV-NEXT: bx lr
;
; CHECK-LIS-LABEL: vld3_v8i32:
; CHECK-LIS: @ %bb.0: @ %entry
; CHECK-LIS-NEXT: .vsave {d8, d9, d10, d11}
; CHECK-LIS-NEXT: vpush {d8, d9, d10, d11}
; CHECK-LIS-NEXT: vldrw.u32 q0, [r0, #64]
; CHECK-LIS-NEXT: vldrw.u32 q1, [r0, #48]
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #80]
; CHECK-LIS-NEXT: vmov.f32 s10, s2
; CHECK-LIS-NEXT: vmov.f32 s17, s0
; CHECK-LIS-NEXT: vmov.f32 s18, s3
; CHECK-LIS-NEXT: vmov.f32 s8, s4
; CHECK-LIS-NEXT: vmov.f32 s9, s7
; CHECK-LIS-NEXT: vmov.f32 s16, s5
; CHECK-LIS-NEXT: vmov.f32 s19, s14
; CHECK-LIS-NEXT: vmov.f32 s11, s13
; CHECK-LIS-NEXT: vmov.f32 s0, s6
; CHECK-LIS-NEXT: vadd.i32 q2, q2, q4
; CHECK-LIS-NEXT: vmov.f32 s2, s12
; CHECK-LIS-NEXT: vldrw.u32 q1, [r0, #16]
; CHECK-LIS-NEXT: vmov.f32 s3, s15
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #32]
; CHECK-LIS-NEXT: vadd.i32 q0, q2, q0
; CHECK-LIS-NEXT: vldrw.u32 q2, [r0]
; CHECK-LIS-NEXT: vmov.f32 s17, s4
; CHECK-LIS-NEXT: vstrw.32 q0, [r1, #16]
; CHECK-LIS-NEXT: vmov.f32 s18, s7
; CHECK-LIS-NEXT: vmov.f32 s22, s6
; CHECK-LIS-NEXT: vmov.f32 s16, s9
; CHECK-LIS-NEXT: vmov.f32 s19, s14
; CHECK-LIS-NEXT: vmov.f32 s20, s8
; CHECK-LIS-NEXT: vmov.f32 s21, s11
; CHECK-LIS-NEXT: vmov.f32 s23, s13
; CHECK-LIS-NEXT: vadd.i32 q4, q5, q4
; CHECK-LIS-NEXT: vmov.f32 s4, s10
; CHECK-LIS-NEXT: vmov.f32 s6, s12
; CHECK-LIS-NEXT: vmov.f32 s7, s15
; CHECK-LIS-NEXT: vadd.i32 q1, q4, q1
; CHECK-LIS-NEXT: vstrw.32 q1, [r1]
; CHECK-LIS-NEXT: vpop {d8, d9, d10, d11}
; CHECK-LIS-NEXT: bx lr
; CHECK-LABEL: vld3_v8i32:
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .vsave {d8, d9, d10, d11}
; CHECK-NEXT: vpush {d8, d9, d10, d11}
; CHECK-NEXT: vldrw.u32 q0, [r0, #64]
; CHECK-NEXT: vldrw.u32 q1, [r0, #48]
; CHECK-NEXT: vldrw.u32 q4, [r0, #80]
; CHECK-NEXT: vmov.f32 s10, s2
; CHECK-NEXT: vmov.f32 s13, s0
; CHECK-NEXT: vmov.f32 s14, s3
; CHECK-NEXT: vmov.f32 s8, s4
; CHECK-NEXT: vmov.f32 s9, s7
; CHECK-NEXT: vmov.f32 s12, s5
; CHECK-NEXT: vmov.f32 s15, s18
; CHECK-NEXT: vmov.f32 s11, s17
; CHECK-NEXT: vadd.i32 q2, q2, q3
; CHECK-NEXT: vmov.f32 s0, s6
; CHECK-NEXT: vmov.f32 s2, s16
; CHECK-NEXT: vldrw.u32 q1, [r0, #16]
; CHECK-NEXT: vmov.f32 s3, s19
; CHECK-NEXT: vldrw.u32 q3, [r0, #32]
; CHECK-NEXT: vadd.i32 q0, q2, q0
; CHECK-NEXT: vldrw.u32 q2, [r0]
; CHECK-NEXT: vmov.f32 s17, s4
; CHECK-NEXT: vstrw.32 q0, [r1, #16]
; CHECK-NEXT: vmov.f32 s18, s7
; CHECK-NEXT: vmov.f32 s22, s6
; CHECK-NEXT: vmov.f32 s16, s9
; CHECK-NEXT: vmov.f32 s19, s14
; CHECK-NEXT: vmov.f32 s20, s8
; CHECK-NEXT: vmov.f32 s21, s11
; CHECK-NEXT: vmov.f32 s23, s13
; CHECK-NEXT: vadd.i32 q4, q5, q4
; CHECK-NEXT: vmov.f32 s4, s10
; CHECK-NEXT: vmov.f32 s6, s12
; CHECK-NEXT: vmov.f32 s7, s15
; CHECK-NEXT: vadd.i32 q1, q4, q1
; CHECK-NEXT: vstrw.32 q1, [r1]
; CHECK-NEXT: vpop {d8, d9, d10, d11}
; CHECK-NEXT: bx lr
entry:
%l1 = load <24 x i32>, ptr %src, align 4
@ -187,155 +122,80 @@ entry:
}
define void @vld3_v16i32(ptr %src, ptr %dst) {
; CHECK-LV-LABEL: vld3_v16i32:
; CHECK-LV: @ %bb.0: @ %entry
; CHECK-LV-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-LV-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-LV-NEXT: vldrw.u32 q0, [r0, #64]
; CHECK-LV-NEXT: vldrw.u32 q1, [r0, #48]
; CHECK-LV-NEXT: vldrw.u32 q4, [r0, #80]
; CHECK-LV-NEXT: vldrw.u32 q6, [r0, #176]
; CHECK-LV-NEXT: vmov.f32 s10, s2
; CHECK-LV-NEXT: vmov.f32 s13, s0
; CHECK-LV-NEXT: vmov.f32 s14, s3
; CHECK-LV-NEXT: vmov.f32 s8, s4
; CHECK-LV-NEXT: vmov.f32 s9, s7
; CHECK-LV-NEXT: vmov.f32 s12, s5
; CHECK-LV-NEXT: vmov.f32 s15, s18
; CHECK-LV-NEXT: vmov.f32 s11, s17
; CHECK-LV-NEXT: vadd.i32 q2, q2, q3
; CHECK-LV-NEXT: vmov.f32 s0, s6
; CHECK-LV-NEXT: vmov.f32 s2, s16
; CHECK-LV-NEXT: vldrw.u32 q1, [r0, #16]
; CHECK-LV-NEXT: vmov.f32 s3, s19
; CHECK-LV-NEXT: vldrw.u32 q3, [r0, #32]
; CHECK-LV-NEXT: vadd.i32 q0, q2, q0
; CHECK-LV-NEXT: vldrw.u32 q2, [r0]
; CHECK-LV-NEXT: vmov.f32 s17, s4
; CHECK-LV-NEXT: vmov.f32 s18, s7
; CHECK-LV-NEXT: vmov.f32 s22, s6
; CHECK-LV-NEXT: vmov.f32 s16, s9
; CHECK-LV-NEXT: vmov.f32 s19, s14
; CHECK-LV-NEXT: vmov.f32 s20, s8
; CHECK-LV-NEXT: vmov.f32 s21, s11
; CHECK-LV-NEXT: vmov.f32 s23, s13
; CHECK-LV-NEXT: vmov.f32 s4, s10
; CHECK-LV-NEXT: vldrw.u32 q2, [r0, #160]
; CHECK-LV-NEXT: vmov.f32 s6, s12
; CHECK-LV-NEXT: vadd.i32 q4, q5, q4
; CHECK-LV-NEXT: vmov.f32 s7, s15
; CHECK-LV-NEXT: vldrw.u32 q3, [r0, #144]
; CHECK-LV-NEXT: vadd.i32 q1, q4, q1
; CHECK-LV-NEXT: vmov.f32 s18, s10
; CHECK-LV-NEXT: vmov.f32 s21, s8
; CHECK-LV-NEXT: vmov.f32 s22, s11
; CHECK-LV-NEXT: vmov.f32 s16, s12
; CHECK-LV-NEXT: vmov.f32 s17, s15
; CHECK-LV-NEXT: vmov.f32 s20, s13
; CHECK-LV-NEXT: vmov.f32 s23, s26
; CHECK-LV-NEXT: vmov.f32 s19, s25
; CHECK-LV-NEXT: vadd.i32 q4, q4, q5
; CHECK-LV-NEXT: vmov.f32 s8, s14
; CHECK-LV-NEXT: vmov.f32 s10, s24
; CHECK-LV-NEXT: vldrw.u32 q3, [r0, #112]
; CHECK-LV-NEXT: vmov.f32 s11, s27
; CHECK-LV-NEXT: vldrw.u32 q5, [r0, #128]
; CHECK-LV-NEXT: vadd.i32 q2, q4, q2
; CHECK-LV-NEXT: vldrw.u32 q4, [r0, #96]
; CHECK-LV-NEXT: vmov.f32 s25, s12
; CHECK-LV-NEXT: vstrw.32 q2, [r1, #48]
; CHECK-LV-NEXT: vmov.f32 s26, s15
; CHECK-LV-NEXT: vstrw.32 q0, [r1, #16]
; CHECK-LV-NEXT: vmov.f32 s30, s14
; CHECK-LV-NEXT: vstrw.32 q1, [r1]
; CHECK-LV-NEXT: vmov.f32 s24, s17
; CHECK-LV-NEXT: vmov.f32 s27, s22
; CHECK-LV-NEXT: vmov.f32 s28, s16
; CHECK-LV-NEXT: vmov.f32 s29, s19
; CHECK-LV-NEXT: vmov.f32 s31, s21
; CHECK-LV-NEXT: vadd.i32 q6, q7, q6
; CHECK-LV-NEXT: vmov.f32 s12, s18
; CHECK-LV-NEXT: vmov.f32 s14, s20
; CHECK-LV-NEXT: vmov.f32 s15, s23
; CHECK-LV-NEXT: vadd.i32 q3, q6, q3
; CHECK-LV-NEXT: vstrw.32 q3, [r1, #32]
; CHECK-LV-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-LV-NEXT: bx lr
;
; CHECK-LIS-LABEL: vld3_v16i32:
; CHECK-LIS: @ %bb.0: @ %entry
; CHECK-LIS-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-LIS-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-LIS-NEXT: vldrw.u32 q0, [r0, #64]
; CHECK-LIS-NEXT: vldrw.u32 q1, [r0, #48]
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #80]
; CHECK-LIS-NEXT: vmov.f32 s10, s2
; CHECK-LIS-NEXT: vmov.f32 s17, s0
; CHECK-LIS-NEXT: vmov.f32 s18, s3
; CHECK-LIS-NEXT: vmov.f32 s8, s4
; CHECK-LIS-NEXT: vmov.f32 s9, s7
; CHECK-LIS-NEXT: vmov.f32 s16, s5
; CHECK-LIS-NEXT: vmov.f32 s19, s14
; CHECK-LIS-NEXT: vmov.f32 s11, s13
; CHECK-LIS-NEXT: vmov.f32 s0, s6
; CHECK-LIS-NEXT: vadd.i32 q2, q2, q4
; CHECK-LIS-NEXT: vmov.f32 s2, s12
; CHECK-LIS-NEXT: vldrw.u32 q1, [r0, #16]
; CHECK-LIS-NEXT: vmov.f32 s3, s15
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #32]
; CHECK-LIS-NEXT: vadd.i32 q0, q2, q0
; CHECK-LIS-NEXT: vldrw.u32 q2, [r0]
; CHECK-LIS-NEXT: vmov.f32 s17, s4
; CHECK-LIS-NEXT: vmov.f32 s18, s7
; CHECK-LIS-NEXT: vmov.f32 s22, s6
; CHECK-LIS-NEXT: vmov.f32 s16, s9
; CHECK-LIS-NEXT: vmov.f32 s19, s14
; CHECK-LIS-NEXT: vmov.f32 s20, s8
; CHECK-LIS-NEXT: vmov.f32 s21, s11
; CHECK-LIS-NEXT: vmov.f32 s23, s13
; CHECK-LIS-NEXT: vadd.i32 q4, q5, q4
; CHECK-LIS-NEXT: vmov.f32 s4, s10
; CHECK-LIS-NEXT: vldrw.u32 q2, [r0, #160]
; CHECK-LIS-NEXT: vldrw.u32 q5, [r0, #176]
; CHECK-LIS-NEXT: vmov.f32 s6, s12
; CHECK-LIS-NEXT: vmov.f32 s7, s15
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #144]
; CHECK-LIS-NEXT: vadd.i32 q1, q4, q1
; CHECK-LIS-NEXT: vmov.f32 s18, s10
; CHECK-LIS-NEXT: vmov.f32 s25, s8
; CHECK-LIS-NEXT: vmov.f32 s26, s11
; CHECK-LIS-NEXT: vmov.f32 s16, s12
; CHECK-LIS-NEXT: vmov.f32 s17, s15
; CHECK-LIS-NEXT: vmov.f32 s24, s13
; CHECK-LIS-NEXT: vmov.f32 s27, s22
; CHECK-LIS-NEXT: vmov.f32 s19, s21
; CHECK-LIS-NEXT: vmov.f32 s8, s14
; CHECK-LIS-NEXT: vadd.i32 q4, q4, q6
; CHECK-LIS-NEXT: vmov.f32 s10, s20
; CHECK-LIS-NEXT: vldrw.u32 q3, [r0, #112]
; CHECK-LIS-NEXT: vmov.f32 s11, s23
; CHECK-LIS-NEXT: vldrw.u32 q5, [r0, #128]
; CHECK-LIS-NEXT: vadd.i32 q2, q4, q2
; CHECK-LIS-NEXT: vldrw.u32 q4, [r0, #96]
; CHECK-LIS-NEXT: vmov.f32 s25, s12
; CHECK-LIS-NEXT: vstrw.32 q2, [r1, #48]
; CHECK-LIS-NEXT: vmov.f32 s26, s15
; CHECK-LIS-NEXT: vstrw.32 q0, [r1, #16]
; CHECK-LIS-NEXT: vmov.f32 s30, s14
; CHECK-LIS-NEXT: vstrw.32 q1, [r1]
; CHECK-LIS-NEXT: vmov.f32 s24, s17
; CHECK-LIS-NEXT: vmov.f32 s27, s22
; CHECK-LIS-NEXT: vmov.f32 s28, s16
; CHECK-LIS-NEXT: vmov.f32 s29, s19
; CHECK-LIS-NEXT: vmov.f32 s31, s21
; CHECK-LIS-NEXT: vadd.i32 q6, q7, q6
; CHECK-LIS-NEXT: vmov.f32 s12, s18
; CHECK-LIS-NEXT: vmov.f32 s14, s20
; CHECK-LIS-NEXT: vmov.f32 s15, s23
; CHECK-LIS-NEXT: vadd.i32 q3, q6, q3
; CHECK-LIS-NEXT: vstrw.32 q3, [r1, #32]
; CHECK-LIS-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-LIS-NEXT: bx lr
; CHECK-LABEL: vld3_v16i32:
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: vldrw.u32 q0, [r0, #64]
; CHECK-NEXT: vldrw.u32 q1, [r0, #48]
; CHECK-NEXT: vldrw.u32 q4, [r0, #80]
; CHECK-NEXT: vldrw.u32 q6, [r0, #176]
; CHECK-NEXT: vmov.f32 s10, s2
; CHECK-NEXT: vmov.f32 s13, s0
; CHECK-NEXT: vmov.f32 s14, s3
; CHECK-NEXT: vmov.f32 s8, s4
; CHECK-NEXT: vmov.f32 s9, s7
; CHECK-NEXT: vmov.f32 s12, s5
; CHECK-NEXT: vmov.f32 s15, s18
; CHECK-NEXT: vmov.f32 s11, s17
; CHECK-NEXT: vadd.i32 q2, q2, q3
; CHECK-NEXT: vmov.f32 s0, s6
; CHECK-NEXT: vmov.f32 s2, s16
; CHECK-NEXT: vldrw.u32 q1, [r0, #16]
; CHECK-NEXT: vmov.f32 s3, s19
; CHECK-NEXT: vldrw.u32 q3, [r0, #32]
; CHECK-NEXT: vadd.i32 q0, q2, q0
; CHECK-NEXT: vldrw.u32 q2, [r0]
; CHECK-NEXT: vmov.f32 s17, s4
; CHECK-NEXT: vmov.f32 s18, s7
; CHECK-NEXT: vmov.f32 s22, s6
; CHECK-NEXT: vmov.f32 s16, s9
; CHECK-NEXT: vmov.f32 s19, s14
; CHECK-NEXT: vmov.f32 s20, s8
; CHECK-NEXT: vmov.f32 s21, s11
; CHECK-NEXT: vmov.f32 s23, s13
; CHECK-NEXT: vmov.f32 s4, s10
; CHECK-NEXT: vldrw.u32 q2, [r0, #160]
; CHECK-NEXT: vmov.f32 s6, s12
; CHECK-NEXT: vadd.i32 q4, q5, q4
; CHECK-NEXT: vmov.f32 s7, s15
; CHECK-NEXT: vldrw.u32 q3, [r0, #144]
; CHECK-NEXT: vadd.i32 q1, q4, q1
; CHECK-NEXT: vmov.f32 s18, s10
; CHECK-NEXT: vmov.f32 s21, s8
; CHECK-NEXT: vmov.f32 s22, s11
; CHECK-NEXT: vmov.f32 s16, s12
; CHECK-NEXT: vmov.f32 s17, s15
; CHECK-NEXT: vmov.f32 s20, s13
; CHECK-NEXT: vmov.f32 s23, s26
; CHECK-NEXT: vmov.f32 s19, s25
; CHECK-NEXT: vadd.i32 q4, q4, q5
; CHECK-NEXT: vmov.f32 s8, s14
; CHECK-NEXT: vmov.f32 s10, s24
; CHECK-NEXT: vldrw.u32 q3, [r0, #112]
; CHECK-NEXT: vmov.f32 s11, s27
; CHECK-NEXT: vldrw.u32 q5, [r0, #128]
; CHECK-NEXT: vadd.i32 q2, q4, q2
; CHECK-NEXT: vldrw.u32 q4, [r0, #96]
; CHECK-NEXT: vmov.f32 s25, s12
; CHECK-NEXT: vstrw.32 q2, [r1, #48]
; CHECK-NEXT: vmov.f32 s26, s15
; CHECK-NEXT: vstrw.32 q0, [r1, #16]
; CHECK-NEXT: vmov.f32 s30, s14
; CHECK-NEXT: vstrw.32 q1, [r1]
; CHECK-NEXT: vmov.f32 s24, s17
; CHECK-NEXT: vmov.f32 s27, s22
; CHECK-NEXT: vmov.f32 s28, s16
; CHECK-NEXT: vmov.f32 s29, s19
; CHECK-NEXT: vmov.f32 s31, s21
; CHECK-NEXT: vadd.i32 q6, q7, q6
; CHECK-NEXT: vmov.f32 s12, s18
; CHECK-NEXT: vmov.f32 s14, s20
; CHECK-NEXT: vmov.f32 s15, s23
; CHECK-NEXT: vadd.i32 q3, q6, q3
; CHECK-NEXT: vstrw.32 q3, [r1, #32]
; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: bx lr
entry:
%l1 = load <48 x i32>, ptr %src, align 4