28d4e33b65
On GFX12+, GLOBAL_INV increments the loadcnt counter but does not write results to any VGPRs. Previously, we unconditionally inserted s_wait_loadcnt 0 at function returns even when the only pending loadcnt was from GLOBAL_INV instructions. This patch optimizes waitcnt insertion by skipping the loadcnt wait at function boundaries when no VGPRs have pending loads. This is determined by checking if any VGPR has a score greater than the lower bound for LOAD_CNT - if not, the pending loadcnt must be from non-VGPR-writing instructions like GLOBAL_INV. The optimization is limited to GFX12+ targets where GLOBAL_INV exists and uses the extended wait count instructions. This is a follow-up optimization to PR #135340 which added tracking for GLOBAL_INV in the waitcnt pass.
267 lines
13 KiB
LLVM
267 lines
13 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
|
|
; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
|
|
; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11 %s
|
|
|
|
|
|
; --------------------------------------------------------------------
|
|
; i32
|
|
; --------------------------------------------------------------------
|
|
|
|
define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, i32 %val) #0 {
|
|
; GFX12-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; GFX12-NEXT: s_wait_expcnt 0x0
|
|
; GFX12-NEXT: s_wait_samplecnt 0x0
|
|
; GFX12-NEXT: s_wait_bvhcnt 0x0
|
|
; GFX12-NEXT: s_wait_kmcnt 0x0
|
|
; GFX12-NEXT: v_mov_b32_e32 v1, s16
|
|
; GFX12-NEXT: s_wait_storecnt 0x0
|
|
; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
|
|
; GFX12-NEXT: s_wait_loadcnt 0x0
|
|
; GFX12-NEXT: global_inv scope:SCOPE_DEV
|
|
; GFX12-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
|
|
; GFX11-NEXT: v_mov_b32_e32 v0, s16
|
|
; GFX11-NEXT: s_mov_b32 s4, 0
|
|
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
|
|
; GFX11-NEXT: .LBB0_1: ; %atomicrmw.start
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: v_mov_b32_e32 v5, v0
|
|
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
|
|
; GFX11-NEXT: v_sub_nc_u32_e32 v0, v5, v2
|
|
; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v5, v2
|
|
; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v0, vcc_lo
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
|
|
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: buffer_gl1_inv
|
|
; GFX11-NEXT: buffer_gl0_inv
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
|
|
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
|
|
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB0_1
|
|
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i32, ptr addrspace(7) %ptr, i32 256
|
|
%result = atomicrmw usub_cond ptr addrspace(7) %gep, i32 %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i32 %result
|
|
}
|
|
|
|
define void @buffer_fat_ptr_agent_atomic_usub_cond_noret_u32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, i32 %val) #0 {
|
|
; GFX12-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_noret_u32__offset__amdgpu_no_fine_grained_memory:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; GFX12-NEXT: s_wait_expcnt 0x0
|
|
; GFX12-NEXT: s_wait_samplecnt 0x0
|
|
; GFX12-NEXT: s_wait_bvhcnt 0x0
|
|
; GFX12-NEXT: s_wait_kmcnt 0x0
|
|
; GFX12-NEXT: v_mov_b32_e32 v1, s16
|
|
; GFX12-NEXT: s_wait_storecnt 0x0
|
|
; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
|
|
; GFX12-NEXT: s_wait_loadcnt 0x0
|
|
; GFX12-NEXT: global_inv scope:SCOPE_DEV
|
|
; GFX12-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_noret_u32__offset__amdgpu_no_fine_grained_memory:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mov_b32_e32 v1, s16
|
|
; GFX11-NEXT: v_mov_b32_e32 v3, s16
|
|
; GFX11-NEXT: s_mov_b32 s4, 0
|
|
; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen offset:1024
|
|
; GFX11-NEXT: .LBB1_1: ; %atomicrmw.start
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: v_sub_nc_u32_e32 v1, v2, v0
|
|
; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v2, v0
|
|
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
|
|
; GFX11-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
|
|
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: buffer_gl1_inv
|
|
; GFX11-NEXT: buffer_gl0_inv
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
|
|
; GFX11-NEXT: v_mov_b32_e32 v2, v4
|
|
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
|
|
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB1_1
|
|
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i32, ptr addrspace(7) %ptr, i32 256
|
|
%unused = atomicrmw usub_cond ptr addrspace(7) %gep, i32 %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, i32 %val) #0 {
|
|
; GFX12-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_remote_memory:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; GFX12-NEXT: s_wait_expcnt 0x0
|
|
; GFX12-NEXT: s_wait_samplecnt 0x0
|
|
; GFX12-NEXT: s_wait_bvhcnt 0x0
|
|
; GFX12-NEXT: s_wait_kmcnt 0x0
|
|
; GFX12-NEXT: v_mov_b32_e32 v1, s16
|
|
; GFX12-NEXT: s_wait_storecnt 0x0
|
|
; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
|
|
; GFX12-NEXT: s_wait_loadcnt 0x0
|
|
; GFX12-NEXT: global_inv scope:SCOPE_DEV
|
|
; GFX12-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_remote_memory:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
|
|
; GFX11-NEXT: v_mov_b32_e32 v0, s16
|
|
; GFX11-NEXT: s_mov_b32 s4, 0
|
|
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
|
|
; GFX11-NEXT: .LBB2_1: ; %atomicrmw.start
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: v_mov_b32_e32 v5, v0
|
|
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
|
|
; GFX11-NEXT: v_sub_nc_u32_e32 v0, v5, v2
|
|
; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v5, v2
|
|
; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v0, vcc_lo
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
|
|
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: buffer_gl1_inv
|
|
; GFX11-NEXT: buffer_gl0_inv
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
|
|
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
|
|
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
|
|
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i32, ptr addrspace(7) %ptr, i32 256
|
|
%result = atomicrmw usub_cond ptr addrspace(7) %gep, i32 %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i32 %result
|
|
}
|
|
|
|
define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, i32 %val) #0 {
|
|
; GFX12-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; GFX12-NEXT: s_wait_expcnt 0x0
|
|
; GFX12-NEXT: s_wait_samplecnt 0x0
|
|
; GFX12-NEXT: s_wait_bvhcnt 0x0
|
|
; GFX12-NEXT: s_wait_kmcnt 0x0
|
|
; GFX12-NEXT: v_mov_b32_e32 v1, s16
|
|
; GFX12-NEXT: s_wait_storecnt 0x0
|
|
; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
|
|
; GFX12-NEXT: s_wait_loadcnt 0x0
|
|
; GFX12-NEXT: global_inv scope:SCOPE_DEV
|
|
; GFX12-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
|
|
; GFX11-NEXT: v_mov_b32_e32 v0, s16
|
|
; GFX11-NEXT: s_mov_b32 s4, 0
|
|
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
|
|
; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: v_mov_b32_e32 v5, v0
|
|
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
|
|
; GFX11-NEXT: v_sub_nc_u32_e32 v0, v5, v2
|
|
; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v5, v2
|
|
; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v0, vcc_lo
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
|
|
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: buffer_gl1_inv
|
|
; GFX11-NEXT: buffer_gl0_inv
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
|
|
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
|
|
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB3_1
|
|
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i32, ptr addrspace(7) %ptr, i32 256
|
|
%result = atomicrmw usub_cond ptr addrspace(7) %gep, i32 %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
|
|
ret i32 %result
|
|
}
|
|
|
|
; --------------------------------------------------------------------
|
|
; misc
|
|
; --------------------------------------------------------------------
|
|
|
|
define i32 @buffer_fat_ptr_system_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, i32 %val) #0 {
|
|
; GFX12-LABEL: buffer_fat_ptr_system_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
|
|
; GFX12-NEXT: s_wait_expcnt 0x0
|
|
; GFX12-NEXT: s_wait_samplecnt 0x0
|
|
; GFX12-NEXT: s_wait_bvhcnt 0x0
|
|
; GFX12-NEXT: s_wait_kmcnt 0x0
|
|
; GFX12-NEXT: v_mov_b32_e32 v1, s16
|
|
; GFX12-NEXT: global_wb scope:SCOPE_SYS
|
|
; GFX12-NEXT: s_wait_storecnt 0x0
|
|
; GFX12-NEXT: buffer_atomic_cond_sub_u32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
|
|
; GFX12-NEXT: s_wait_loadcnt 0x0
|
|
; GFX12-NEXT: global_inv scope:SCOPE_SYS
|
|
; GFX12-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX11-LABEL: buffer_fat_ptr_system_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
|
|
; GFX11-NEXT: v_mov_b32_e32 v0, s16
|
|
; GFX11-NEXT: s_mov_b32 s4, 0
|
|
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
|
|
; GFX11-NEXT: .LBB4_1: ; %atomicrmw.start
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: v_mov_b32_e32 v5, v0
|
|
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
|
|
; GFX11-NEXT: v_sub_nc_u32_e32 v0, v5, v2
|
|
; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v5, v2
|
|
; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v0, vcc_lo
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
|
|
; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
|
; GFX11-NEXT: buffer_gl1_inv
|
|
; GFX11-NEXT: buffer_gl0_inv
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
|
|
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
|
|
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB4_1
|
|
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i32, ptr addrspace(7) %ptr, i32 256
|
|
%result = atomicrmw usub_cond ptr addrspace(7) %gep, i32 %val seq_cst, !amdgpu.no.fine.grained.memory !0
|
|
ret i32 %result
|
|
}
|
|
|
|
attributes #0 = { nounwind }
|
|
|
|
!0 = !{}
|