b1b5102624
Start considering !amdgpu.no.remote.memory.access and !amdgpu.no.fine.grained.host.memory metadata when deciding to expand integer atomic operations. This does not yet attempt to accurately handle fadd/fmin/fmax, which are trickier and require migrating the old "amdgpu-unsafe-fp-atomics" attribute.
25446 lines
1.1 MiB
25446 lines
1.1 MiB
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GCN1 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN2 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN3 %s
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw xchg
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_xchg_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB0_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB0_4
|
|
; GCN1-NEXT: .LBB0_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB0_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB0_2
|
|
; GCN1-NEXT: .LBB0_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB0_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB0_4
|
|
; GCN2-NEXT: .LBB0_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB0_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB0_2
|
|
; GCN2-NEXT: .LBB0_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB0_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB0_4
|
|
; GCN3-NEXT: .LBB0_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB0_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB0_2
|
|
; GCN3-NEXT: .LBB0_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_xchg_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB1_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB1_4
|
|
; GCN1-NEXT: .LBB1_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB1_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB1_2
|
|
; GCN1-NEXT: .LBB1_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB1_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB1_4
|
|
; GCN2-NEXT: .LBB1_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB1_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB1_2
|
|
; GCN2-NEXT: .LBB1_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB1_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB1_4
|
|
; GCN3-NEXT: .LBB1_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB1_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB1_2
|
|
; GCN3-NEXT: .LBB1_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_xchg_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB2_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB2_4
|
|
; GCN1-NEXT: .LBB2_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB2_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB2_2
|
|
; GCN1-NEXT: .LBB2_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB2_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB2_4
|
|
; GCN2-NEXT: .LBB2_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB2_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB2_2
|
|
; GCN2-NEXT: .LBB2_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB2_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB2_4
|
|
; GCN3-NEXT: .LBB2_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB2_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB2_2
|
|
; GCN3-NEXT: .LBB2_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xchg ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_xchg_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB3_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB3_4
|
|
; GCN1-NEXT: .LBB3_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB3_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB3_2
|
|
; GCN1-NEXT: .LBB3_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB3_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB3_4
|
|
; GCN2-NEXT: .LBB3_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB3_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB3_2
|
|
; GCN2-NEXT: .LBB3_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB3_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB3_4
|
|
; GCN3-NEXT: .LBB3_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB3_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB3_2
|
|
; GCN3-NEXT: .LBB3_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB4_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB4_4
|
|
; GCN1-NEXT: .LBB4_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB4_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB4_2
|
|
; GCN1-NEXT: .LBB4_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB4_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB4_4
|
|
; GCN2-NEXT: .LBB4_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB4_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB4_2
|
|
; GCN2-NEXT: .LBB4_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB4_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB4_4
|
|
; GCN3-NEXT: .LBB4_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB4_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB4_2
|
|
; GCN3-NEXT: .LBB4_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB5_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB5_4
|
|
; GCN1-NEXT: .LBB5_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB5_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB5_2
|
|
; GCN1-NEXT: .LBB5_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB5_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB5_4
|
|
; GCN2-NEXT: .LBB5_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB5_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB5_2
|
|
; GCN2-NEXT: .LBB5_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB5_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB5_4
|
|
; GCN3-NEXT: .LBB5_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB5_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB5_2
|
|
; GCN3-NEXT: .LBB5_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB6_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB6_3
|
|
; GCN1-NEXT: s_branch .LBB6_4
|
|
; GCN1-NEXT: .LBB6_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB6_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB6_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB6_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB6_3
|
|
; GCN2-NEXT: s_branch .LBB6_4
|
|
; GCN2-NEXT: .LBB6_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB6_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB6_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB6_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB6_3
|
|
; GCN3-NEXT: s_branch .LBB6_4
|
|
; GCN3-NEXT: .LBB6_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB6_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s6
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB6_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xchg ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB7_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB7_3
|
|
; GCN1-NEXT: s_branch .LBB7_4
|
|
; GCN1-NEXT: .LBB7_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB7_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB7_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB7_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB7_3
|
|
; GCN2-NEXT: s_branch .LBB7_4
|
|
; GCN2-NEXT: .LBB7_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB7_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB7_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB7_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB7_3
|
|
; GCN3-NEXT: s_branch .LBB7_4
|
|
; GCN3-NEXT: .LBB7_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB7_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s6
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB7_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB8_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB8_4
|
|
; GCN1-NEXT: .LBB8_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB8_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB8_2
|
|
; GCN1-NEXT: .LBB8_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB8_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB8_4
|
|
; GCN2-NEXT: .LBB8_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB8_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB8_2
|
|
; GCN2-NEXT: .LBB8_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB8_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB8_4
|
|
; GCN3-NEXT: .LBB8_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB8_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB8_2
|
|
; GCN3-NEXT: .LBB8_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB9_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB9_4
|
|
; GCN1-NEXT: .LBB9_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB9_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB9_2
|
|
; GCN1-NEXT: .LBB9_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB9_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB9_4
|
|
; GCN2-NEXT: .LBB9_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB9_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB9_2
|
|
; GCN2-NEXT: .LBB9_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB9_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB9_4
|
|
; GCN3-NEXT: .LBB9_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB9_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB9_2
|
|
; GCN3-NEXT: .LBB9_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw xchg f64
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_xchg_f64_noret(ptr %ptr, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB10_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB10_4
|
|
; GCN1-NEXT: .LBB10_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB10_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB10_2
|
|
; GCN1-NEXT: .LBB10_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB10_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB10_4
|
|
; GCN2-NEXT: .LBB10_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB10_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB10_2
|
|
; GCN2-NEXT: .LBB10_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB10_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB10_4
|
|
; GCN3-NEXT: .LBB10_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB10_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB10_2
|
|
; GCN3-NEXT: .LBB10_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_xchg_f64_noret_offset(ptr %out, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB11_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB11_4
|
|
; GCN1-NEXT: .LBB11_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB11_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB11_2
|
|
; GCN1-NEXT: .LBB11_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB11_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB11_4
|
|
; GCN2-NEXT: .LBB11_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB11_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB11_2
|
|
; GCN2-NEXT: .LBB11_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB11_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB11_4
|
|
; GCN3-NEXT: .LBB11_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB11_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB11_2
|
|
; GCN3-NEXT: .LBB11_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i32 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define double @flat_atomic_xchg_f64_ret(ptr %ptr, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB12_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB12_4
|
|
; GCN1-NEXT: .LBB12_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB12_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB12_2
|
|
; GCN1-NEXT: .LBB12_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB12_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB12_4
|
|
; GCN2-NEXT: .LBB12_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB12_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB12_2
|
|
; GCN2-NEXT: .LBB12_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB12_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB12_4
|
|
; GCN3-NEXT: .LBB12_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB12_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB12_2
|
|
; GCN3-NEXT: .LBB12_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xchg ptr %ptr, double %in seq_cst
|
|
ret double %result
|
|
}
|
|
|
|
define double @flat_atomic_xchg_f64_ret_offset(ptr %out, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB13_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB13_4
|
|
; GCN1-NEXT: .LBB13_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB13_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB13_2
|
|
; GCN1-NEXT: .LBB13_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB13_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB13_4
|
|
; GCN2-NEXT: .LBB13_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB13_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB13_2
|
|
; GCN2-NEXT: .LBB13_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB13_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB13_4
|
|
; GCN3-NEXT: .LBB13_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB13_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB13_2
|
|
; GCN3-NEXT: .LBB13_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i32 4
|
|
%result = atomicrmw xchg ptr %gep, double %in seq_cst
|
|
ret double %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB14_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB14_4
|
|
; GCN1-NEXT: .LBB14_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB14_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB14_2
|
|
; GCN1-NEXT: .LBB14_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB14_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB14_4
|
|
; GCN2-NEXT: .LBB14_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB14_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB14_2
|
|
; GCN2-NEXT: .LBB14_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB14_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB14_4
|
|
; GCN3-NEXT: .LBB14_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB14_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB14_2
|
|
; GCN3-NEXT: .LBB14_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out, double inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB15_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB15_4
|
|
; GCN1-NEXT: .LBB15_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB15_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB15_2
|
|
; GCN1-NEXT: .LBB15_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB15_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB15_4
|
|
; GCN2-NEXT: .LBB15_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB15_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB15_2
|
|
; GCN2-NEXT: .LBB15_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB15_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB15_4
|
|
; GCN3-NEXT: .LBB15_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB15_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB15_2
|
|
; GCN3-NEXT: .LBB15_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i32 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB16_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB16_3
|
|
; GCN1-NEXT: s_branch .LBB16_4
|
|
; GCN1-NEXT: .LBB16_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB16_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB16_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB16_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB16_3
|
|
; GCN2-NEXT: s_branch .LBB16_4
|
|
; GCN2-NEXT: .LBB16_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB16_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB16_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB16_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB16_3
|
|
; GCN3-NEXT: s_branch .LBB16_4
|
|
; GCN3-NEXT: .LBB16_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB16_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s6
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB16_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xchg ptr %ptr, double %in seq_cst
|
|
ret double %result
|
|
}
|
|
|
|
define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out, double inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB17_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB17_3
|
|
; GCN1-NEXT: s_branch .LBB17_4
|
|
; GCN1-NEXT: .LBB17_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB17_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB17_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB17_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB17_3
|
|
; GCN2-NEXT: s_branch .LBB17_4
|
|
; GCN2-NEXT: .LBB17_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB17_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB17_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB17_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB17_3
|
|
; GCN3-NEXT: s_branch .LBB17_4
|
|
; GCN3-NEXT: .LBB17_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB17_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s6
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB17_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i32 4
|
|
%result = atomicrmw xchg ptr %gep, double %in seq_cst
|
|
ret double %result
|
|
}
|
|
|
|
define void @flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB18_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB18_4
|
|
; GCN1-NEXT: .LBB18_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB18_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB18_2
|
|
; GCN1-NEXT: .LBB18_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB18_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB18_4
|
|
; GCN2-NEXT: .LBB18_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB18_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB18_2
|
|
; GCN2-NEXT: .LBB18_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB18_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB18_4
|
|
; GCN3-NEXT: .LBB18_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB18_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB18_2
|
|
; GCN3-NEXT: .LBB18_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define double @flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB19_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB19_4
|
|
; GCN1-NEXT: .LBB19_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB19_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB19_2
|
|
; GCN1-NEXT: .LBB19_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB19_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB19_4
|
|
; GCN2-NEXT: .LBB19_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB19_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB19_2
|
|
; GCN2-NEXT: .LBB19_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB19_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB19_4
|
|
; GCN3-NEXT: .LBB19_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB19_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB19_2
|
|
; GCN3-NEXT: .LBB19_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i64 4
|
|
%result = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret double %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw add
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_add_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB20_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB20_4
|
|
; GCN1-NEXT: .LBB20_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB20_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB20_2
|
|
; GCN1-NEXT: .LBB20_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB20_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB20_4
|
|
; GCN2-NEXT: .LBB20_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB20_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB20_2
|
|
; GCN2-NEXT: .LBB20_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB20_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB20_4
|
|
; GCN3-NEXT: .LBB20_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB20_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB20_2
|
|
; GCN3-NEXT: .LBB20_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_add_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB21_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB21_4
|
|
; GCN1-NEXT: .LBB21_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB21_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB21_2
|
|
; GCN1-NEXT: .LBB21_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB21_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB21_4
|
|
; GCN2-NEXT: .LBB21_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB21_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB21_2
|
|
; GCN2-NEXT: .LBB21_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB21_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB21_4
|
|
; GCN3-NEXT: .LBB21_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB21_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB21_2
|
|
; GCN3-NEXT: .LBB21_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_add_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB22_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB22_4
|
|
; GCN1-NEXT: .LBB22_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB22_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB22_2
|
|
; GCN1-NEXT: .LBB22_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB22_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB22_4
|
|
; GCN2-NEXT: .LBB22_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB22_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB22_2
|
|
; GCN2-NEXT: .LBB22_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB22_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB22_4
|
|
; GCN3-NEXT: .LBB22_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB22_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB22_2
|
|
; GCN3-NEXT: .LBB22_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw add ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_add_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB23_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB23_4
|
|
; GCN1-NEXT: .LBB23_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB23_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB23_2
|
|
; GCN1-NEXT: .LBB23_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB23_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB23_4
|
|
; GCN2-NEXT: .LBB23_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB23_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB23_2
|
|
; GCN2-NEXT: .LBB23_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB23_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB23_4
|
|
; GCN3-NEXT: .LBB23_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB23_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB23_2
|
|
; GCN3-NEXT: .LBB23_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw add ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB24_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB24_4
|
|
; GCN1-NEXT: .LBB24_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB24_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB24_2
|
|
; GCN1-NEXT: .LBB24_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB24_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB24_4
|
|
; GCN2-NEXT: .LBB24_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB24_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB24_2
|
|
; GCN2-NEXT: .LBB24_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB24_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB24_4
|
|
; GCN3-NEXT: .LBB24_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB24_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB24_2
|
|
; GCN3-NEXT: .LBB24_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB25_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB25_4
|
|
; GCN1-NEXT: .LBB25_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB25_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB25_2
|
|
; GCN1-NEXT: .LBB25_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB25_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB25_4
|
|
; GCN2-NEXT: .LBB25_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB25_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB25_2
|
|
; GCN2-NEXT: .LBB25_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB25_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB25_4
|
|
; GCN3-NEXT: .LBB25_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB25_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB25_2
|
|
; GCN3-NEXT: .LBB25_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB26_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB26_3
|
|
; GCN1-NEXT: s_branch .LBB26_4
|
|
; GCN1-NEXT: .LBB26_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB26_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB26_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB26_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB26_3
|
|
; GCN2-NEXT: s_branch .LBB26_4
|
|
; GCN2-NEXT: .LBB26_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB26_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB26_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB26_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB26_3
|
|
; GCN3-NEXT: s_branch .LBB26_4
|
|
; GCN3-NEXT: .LBB26_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB26_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB26_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw add ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB27_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB27_3
|
|
; GCN1-NEXT: s_branch .LBB27_4
|
|
; GCN1-NEXT: .LBB27_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB27_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB27_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB27_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB27_3
|
|
; GCN2-NEXT: s_branch .LBB27_4
|
|
; GCN2-NEXT: .LBB27_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB27_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB27_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB27_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB27_3
|
|
; GCN3-NEXT: s_branch .LBB27_4
|
|
; GCN3-NEXT: .LBB27_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB27_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB27_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw add ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB28_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB28_4
|
|
; GCN1-NEXT: .LBB28_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB28_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB28_2
|
|
; GCN1-NEXT: .LBB28_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB28_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB28_4
|
|
; GCN2-NEXT: .LBB28_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB28_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB28_2
|
|
; GCN2-NEXT: .LBB28_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB28_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB28_4
|
|
; GCN3-NEXT: .LBB28_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB28_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB28_2
|
|
; GCN3-NEXT: .LBB28_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB29_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB29_4
|
|
; GCN1-NEXT: .LBB29_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB29_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB29_2
|
|
; GCN1-NEXT: .LBB29_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB29_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB29_4
|
|
; GCN2-NEXT: .LBB29_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB29_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB29_2
|
|
; GCN2-NEXT: .LBB29_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB29_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB29_4
|
|
; GCN3-NEXT: .LBB29_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB29_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB29_2
|
|
; GCN3-NEXT: .LBB29_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw sub
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB30_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB30_6
|
|
; GCN1-NEXT: .LBB30_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB30_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB30_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_sub_i32_e32 v4, vcc, v6, v2
|
|
; GCN1-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB30_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB30_2
|
|
; GCN1-NEXT: .LBB30_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB30_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB30_6
|
|
; GCN2-NEXT: .LBB30_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB30_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB30_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_sub_u32_e32 v4, vcc, v6, v2
|
|
; GCN2-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB30_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB30_2
|
|
; GCN2-NEXT: .LBB30_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB30_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB30_6
|
|
; GCN3-NEXT: .LBB30_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB30_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB30_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v2
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB30_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB30_2
|
|
; GCN3-NEXT: .LBB30_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB31_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB31_6
|
|
; GCN1-NEXT: .LBB31_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB31_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB31_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_sub_i32_e32 v4, vcc, v6, v2
|
|
; GCN1-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB31_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB31_2
|
|
; GCN1-NEXT: .LBB31_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB31_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB31_6
|
|
; GCN2-NEXT: .LBB31_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB31_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB31_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_sub_u32_e32 v4, vcc, v6, v2
|
|
; GCN2-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB31_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB31_2
|
|
; GCN2-NEXT: .LBB31_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB31_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB31_6
|
|
; GCN3-NEXT: .LBB31_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB31_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB31_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v2
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB31_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB31_2
|
|
; GCN3-NEXT: .LBB31_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB32_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB32_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_sub_i32_e32 v4, vcc, v6, v2
|
|
; GCN1-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB32_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: .LBB32_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB32_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB32_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB32_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB32_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_sub_u32_e32 v4, vcc, v6, v2
|
|
; GCN2-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB32_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: .LBB32_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB32_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB32_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB32_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB32_6
|
|
; GCN3-NEXT: .LBB32_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB32_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB32_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v6, vcc, v8, v2
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v3, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB32_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB32_2
|
|
; GCN3-NEXT: .LBB32_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw sub ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB33_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB33_6
|
|
; GCN1-NEXT: .LBB33_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB33_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB33_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_sub_i32_e32 v6, vcc, v8, v2
|
|
; GCN1-NEXT: v_subb_u32_e32 v7, vcc, v9, v3, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB33_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB33_2
|
|
; GCN1-NEXT: .LBB33_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB33_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB33_6
|
|
; GCN2-NEXT: .LBB33_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB33_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB33_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_sub_u32_e32 v6, vcc, v8, v2
|
|
; GCN2-NEXT: v_subb_u32_e32 v7, vcc, v9, v3, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB33_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB33_2
|
|
; GCN2-NEXT: .LBB33_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB33_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB33_6
|
|
; GCN3-NEXT: .LBB33_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB33_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB33_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v6, vcc, v8, v2
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v3, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB33_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB33_2
|
|
; GCN3-NEXT: .LBB33_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw sub ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB34_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB34_6
|
|
; GCN1-NEXT: .LBB34_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB34_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: .LBB34_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
|
|
; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB34_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB34_2
|
|
; GCN1-NEXT: .LBB34_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB34_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB34_6
|
|
; GCN2-NEXT: .LBB34_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB34_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: .LBB34_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
|
|
; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB34_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB34_2
|
|
; GCN2-NEXT: .LBB34_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB34_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB34_6
|
|
; GCN3-NEXT: .LBB34_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB34_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: .LBB34_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB34_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB34_2
|
|
; GCN3-NEXT: .LBB34_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB35_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB35_6
|
|
; GCN1-NEXT: .LBB35_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB35_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: .LBB35_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
|
|
; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB35_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB35_2
|
|
; GCN1-NEXT: .LBB35_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB35_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB35_6
|
|
; GCN2-NEXT: .LBB35_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB35_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: .LBB35_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
|
|
; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB35_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB35_2
|
|
; GCN2-NEXT: .LBB35_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB35_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB35_6
|
|
; GCN3-NEXT: .LBB35_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB35_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: .LBB35_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB35_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB35_2
|
|
; GCN3-NEXT: .LBB35_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB36_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: .LBB36_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v0
|
|
; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v7
|
|
; GCN1-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB36_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB36_6
|
|
; GCN1-NEXT: .LBB36_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB36_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB36_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB36_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: .LBB36_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v0
|
|
; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7
|
|
; GCN2-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB36_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB36_6
|
|
; GCN2-NEXT: .LBB36_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB36_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB36_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB36_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: .LBB36_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v0
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v7
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB36_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB36_6
|
|
; GCN3-NEXT: .LBB36_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB36_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB36_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw sub ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB37_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: .LBB37_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v0
|
|
; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v7
|
|
; GCN1-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB37_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB37_6
|
|
; GCN1-NEXT: .LBB37_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB37_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB37_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB37_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: .LBB37_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v0
|
|
; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7
|
|
; GCN2-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB37_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB37_6
|
|
; GCN2-NEXT: .LBB37_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB37_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB37_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB37_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: .LBB37_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v0
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v7
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB37_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB37_6
|
|
; GCN3-NEXT: .LBB37_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB37_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB37_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw sub ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB38_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB38_4
|
|
; GCN1-NEXT: .LBB38_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB38_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB38_2
|
|
; GCN1-NEXT: .LBB38_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB38_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB38_4
|
|
; GCN2-NEXT: .LBB38_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB38_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB38_2
|
|
; GCN2-NEXT: .LBB38_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB38_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB38_4
|
|
; GCN3-NEXT: .LBB38_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB38_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB38_2
|
|
; GCN3-NEXT: .LBB38_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB39_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB39_4
|
|
; GCN1-NEXT: .LBB39_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB39_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB39_2
|
|
; GCN1-NEXT: .LBB39_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB39_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB39_4
|
|
; GCN2-NEXT: .LBB39_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB39_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB39_2
|
|
; GCN2-NEXT: .LBB39_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB39_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB39_4
|
|
; GCN3-NEXT: .LBB39_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB39_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB39_2
|
|
; GCN3-NEXT: .LBB39_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw and
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB40_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB40_6
|
|
; GCN1-NEXT: .LBB40_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB40_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB40_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB40_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB40_2
|
|
; GCN1-NEXT: .LBB40_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB40_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB40_6
|
|
; GCN2-NEXT: .LBB40_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB40_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB40_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB40_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB40_2
|
|
; GCN2-NEXT: .LBB40_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB40_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB40_6
|
|
; GCN3-NEXT: .LBB40_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB40_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB40_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v5, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB40_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB40_2
|
|
; GCN3-NEXT: .LBB40_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB41_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB41_6
|
|
; GCN1-NEXT: .LBB41_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB41_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB41_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB41_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB41_2
|
|
; GCN1-NEXT: .LBB41_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB41_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB41_6
|
|
; GCN2-NEXT: .LBB41_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB41_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB41_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB41_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB41_2
|
|
; GCN2-NEXT: .LBB41_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB41_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB41_6
|
|
; GCN3-NEXT: .LBB41_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB41_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB41_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v5, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB41_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB41_2
|
|
; GCN3-NEXT: .LBB41_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB42_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB42_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_and_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB42_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: .LBB42_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB42_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB42_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB42_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB42_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_and_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB42_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: .LBB42_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB42_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB42_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB42_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB42_6
|
|
; GCN3-NEXT: .LBB42_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB42_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB42_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v7, v9, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v6, v8, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB42_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB42_2
|
|
; GCN3-NEXT: .LBB42_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw and ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB43_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB43_6
|
|
; GCN1-NEXT: .LBB43_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB43_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB43_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v7, v9, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v6, v8, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB43_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB43_2
|
|
; GCN1-NEXT: .LBB43_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB43_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB43_6
|
|
; GCN2-NEXT: .LBB43_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB43_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB43_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v7, v9, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v6, v8, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB43_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB43_2
|
|
; GCN2-NEXT: .LBB43_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB43_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB43_6
|
|
; GCN3-NEXT: .LBB43_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB43_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB43_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v7, v9, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v6, v8, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB43_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB43_2
|
|
; GCN3-NEXT: .LBB43_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw and ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB44_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB44_6
|
|
; GCN1-NEXT: .LBB44_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB44_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB44_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v1, s7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB44_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB44_2
|
|
; GCN1-NEXT: .LBB44_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB44_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB44_6
|
|
; GCN2-NEXT: .LBB44_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB44_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB44_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v1, s7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB44_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB44_2
|
|
; GCN2-NEXT: .LBB44_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB44_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB44_6
|
|
; GCN3-NEXT: .LBB44_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB44_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB44_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB44_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB44_2
|
|
; GCN3-NEXT: .LBB44_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB45_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB45_6
|
|
; GCN1-NEXT: .LBB45_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB45_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB45_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v1, s7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB45_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB45_2
|
|
; GCN1-NEXT: .LBB45_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB45_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB45_6
|
|
; GCN2-NEXT: .LBB45_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB45_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB45_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v1, s7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB45_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB45_2
|
|
; GCN2-NEXT: .LBB45_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB45_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB45_6
|
|
; GCN3-NEXT: .LBB45_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB45_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB45_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB45_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB45_2
|
|
; GCN3-NEXT: .LBB45_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB46_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB46_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v7
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB46_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB46_6
|
|
; GCN1-NEXT: .LBB46_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB46_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB46_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB46_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB46_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v7
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB46_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB46_6
|
|
; GCN2-NEXT: .LBB46_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB46_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB46_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB46_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB46_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v5, s7, v7
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB46_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB46_6
|
|
; GCN3-NEXT: .LBB46_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB46_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB46_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw and ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB47_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB47_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v7
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB47_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB47_6
|
|
; GCN1-NEXT: .LBB47_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB47_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB47_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB47_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB47_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v7
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB47_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB47_6
|
|
; GCN2-NEXT: .LBB47_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB47_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB47_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB47_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB47_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v5, s7, v7
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB47_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB47_6
|
|
; GCN3-NEXT: .LBB47_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB47_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB47_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw and ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB48_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB48_4
|
|
; GCN1-NEXT: .LBB48_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB48_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB48_2
|
|
; GCN1-NEXT: .LBB48_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB48_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB48_4
|
|
; GCN2-NEXT: .LBB48_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB48_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB48_2
|
|
; GCN2-NEXT: .LBB48_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB48_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB48_4
|
|
; GCN3-NEXT: .LBB48_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB48_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB48_2
|
|
; GCN3-NEXT: .LBB48_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB49_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB49_4
|
|
; GCN1-NEXT: .LBB49_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB49_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB49_2
|
|
; GCN1-NEXT: .LBB49_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB49_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB49_4
|
|
; GCN2-NEXT: .LBB49_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB49_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB49_2
|
|
; GCN2-NEXT: .LBB49_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB49_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB49_4
|
|
; GCN3-NEXT: .LBB49_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB49_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB49_2
|
|
; GCN3-NEXT: .LBB49_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw nand
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_nand_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB50_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB50_6
|
|
; GCN1-NEXT: .LBB50_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB50_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB50_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB50_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB50_2
|
|
; GCN1-NEXT: .LBB50_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB50_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB50_6
|
|
; GCN2-NEXT: .LBB50_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB50_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB50_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB50_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB50_2
|
|
; GCN2-NEXT: .LBB50_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB50_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB50_6
|
|
; GCN3-NEXT: .LBB50_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB50_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB50_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB50_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB50_2
|
|
; GCN3-NEXT: .LBB50_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_nand_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB51_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB51_6
|
|
; GCN1-NEXT: .LBB51_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB51_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB51_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB51_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB51_2
|
|
; GCN1-NEXT: .LBB51_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB51_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB51_6
|
|
; GCN2-NEXT: .LBB51_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB51_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB51_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB51_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB51_2
|
|
; GCN2-NEXT: .LBB51_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB51_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB51_6
|
|
; GCN3-NEXT: .LBB51_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB51_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB51_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB51_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB51_2
|
|
; GCN3-NEXT: .LBB51_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_nand_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB52_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB52_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB52_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: .LBB52_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB52_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB52_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB52_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB52_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB52_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: .LBB52_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB52_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB52_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB52_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB52_6
|
|
; GCN3-NEXT: .LBB52_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB52_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB52_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB52_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB52_2
|
|
; GCN3-NEXT: .LBB52_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw nand ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_nand_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB53_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB53_6
|
|
; GCN1-NEXT: .LBB53_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB53_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB53_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB53_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB53_2
|
|
; GCN1-NEXT: .LBB53_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB53_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB53_6
|
|
; GCN2-NEXT: .LBB53_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB53_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB53_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB53_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB53_2
|
|
; GCN2-NEXT: .LBB53_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB53_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB53_6
|
|
; GCN3-NEXT: .LBB53_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB53_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB53_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB53_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB53_2
|
|
; GCN3-NEXT: .LBB53_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw nand ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB54_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB54_6
|
|
; GCN1-NEXT: .LBB54_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB54_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB54_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB54_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB54_2
|
|
; GCN1-NEXT: .LBB54_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB54_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB54_6
|
|
; GCN2-NEXT: .LBB54_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB54_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB54_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB54_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB54_2
|
|
; GCN2-NEXT: .LBB54_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB54_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB54_6
|
|
; GCN3-NEXT: .LBB54_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB54_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB54_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB54_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB54_2
|
|
; GCN3-NEXT: .LBB54_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB55_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB55_6
|
|
; GCN1-NEXT: .LBB55_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB55_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB55_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB55_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB55_2
|
|
; GCN1-NEXT: .LBB55_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB55_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB55_6
|
|
; GCN2-NEXT: .LBB55_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB55_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB55_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB55_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB55_2
|
|
; GCN2-NEXT: .LBB55_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB55_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB55_6
|
|
; GCN3-NEXT: .LBB55_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB55_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB55_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB55_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB55_2
|
|
; GCN3-NEXT: .LBB55_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB56_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB56_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB56_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB56_6
|
|
; GCN1-NEXT: .LBB56_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB56_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v5
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB56_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB56_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB56_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB56_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB56_6
|
|
; GCN2-NEXT: .LBB56_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB56_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v5
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB56_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB56_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB56_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB56_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB56_6
|
|
; GCN3-NEXT: .LBB56_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB56_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB56_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw nand ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB57_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB57_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB57_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB57_6
|
|
; GCN1-NEXT: .LBB57_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB57_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v5
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB57_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB57_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB57_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB57_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB57_6
|
|
; GCN2-NEXT: .LBB57_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB57_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v5
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB57_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB57_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB57_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB57_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB57_6
|
|
; GCN3-NEXT: .LBB57_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB57_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB57_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw nand ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB58_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB58_6
|
|
; GCN1-NEXT: .LBB58_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB58_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB58_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB58_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB58_2
|
|
; GCN1-NEXT: .LBB58_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB58_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB58_6
|
|
; GCN2-NEXT: .LBB58_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB58_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB58_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB58_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB58_2
|
|
; GCN2-NEXT: .LBB58_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB58_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB58_6
|
|
; GCN3-NEXT: .LBB58_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB58_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB58_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB58_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB58_2
|
|
; GCN3-NEXT: .LBB58_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB59_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB59_6
|
|
; GCN1-NEXT: .LBB59_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB59_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB59_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB59_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB59_2
|
|
; GCN1-NEXT: .LBB59_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB59_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB59_6
|
|
; GCN2-NEXT: .LBB59_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB59_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB59_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB59_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB59_2
|
|
; GCN2-NEXT: .LBB59_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB59_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB59_6
|
|
; GCN3-NEXT: .LBB59_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB59_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB59_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB59_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB59_2
|
|
; GCN3-NEXT: .LBB59_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw or
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB60_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB60_6
|
|
; GCN1-NEXT: .LBB60_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB60_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB60_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_or_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB60_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB60_2
|
|
; GCN1-NEXT: .LBB60_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB60_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB60_6
|
|
; GCN2-NEXT: .LBB60_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB60_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB60_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_or_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB60_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB60_2
|
|
; GCN2-NEXT: .LBB60_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB60_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB60_6
|
|
; GCN3-NEXT: .LBB60_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB60_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB60_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v5, v7, v3
|
|
; GCN3-NEXT: v_or_b32_e32 v4, v6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB60_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB60_2
|
|
; GCN3-NEXT: .LBB60_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB61_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB61_6
|
|
; GCN1-NEXT: .LBB61_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB61_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB61_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_or_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB61_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB61_2
|
|
; GCN1-NEXT: .LBB61_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB61_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB61_6
|
|
; GCN2-NEXT: .LBB61_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB61_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB61_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_or_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB61_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB61_2
|
|
; GCN2-NEXT: .LBB61_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB61_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB61_6
|
|
; GCN3-NEXT: .LBB61_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB61_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB61_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v5, v7, v3
|
|
; GCN3-NEXT: v_or_b32_e32 v4, v6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB61_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB61_2
|
|
; GCN3-NEXT: .LBB61_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB62_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB62_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_or_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_or_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB62_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: .LBB62_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB62_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB62_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB62_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB62_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_or_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_or_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB62_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: .LBB62_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB62_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB62_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB62_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB62_6
|
|
; GCN3-NEXT: .LBB62_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB62_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB62_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_or_b32_e32 v7, v9, v3
|
|
; GCN3-NEXT: v_or_b32_e32 v6, v8, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB62_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB62_2
|
|
; GCN3-NEXT: .LBB62_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw or ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB63_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB63_6
|
|
; GCN1-NEXT: .LBB63_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB63_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB63_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_or_b32_e32 v7, v9, v3
|
|
; GCN1-NEXT: v_or_b32_e32 v6, v8, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB63_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB63_2
|
|
; GCN1-NEXT: .LBB63_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB63_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB63_6
|
|
; GCN2-NEXT: .LBB63_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB63_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB63_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_or_b32_e32 v7, v9, v3
|
|
; GCN2-NEXT: v_or_b32_e32 v6, v8, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB63_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB63_2
|
|
; GCN2-NEXT: .LBB63_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB63_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB63_6
|
|
; GCN3-NEXT: .LBB63_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB63_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB63_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_or_b32_e32 v7, v9, v3
|
|
; GCN3-NEXT: v_or_b32_e32 v6, v8, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB63_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB63_2
|
|
; GCN3-NEXT: .LBB63_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw or ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB64_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB64_6
|
|
; GCN1-NEXT: .LBB64_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB64_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB64_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v1, s7, v3
|
|
; GCN1-NEXT: v_or_b32_e32 v0, s6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB64_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB64_2
|
|
; GCN1-NEXT: .LBB64_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB64_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB64_6
|
|
; GCN2-NEXT: .LBB64_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB64_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB64_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v1, s7, v3
|
|
; GCN2-NEXT: v_or_b32_e32 v0, s6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB64_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB64_2
|
|
; GCN2-NEXT: .LBB64_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB64_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB64_6
|
|
; GCN3-NEXT: .LBB64_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB64_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB64_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, s7, v3
|
|
; GCN3-NEXT: v_or_b32_e32 v0, s6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB64_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB64_2
|
|
; GCN3-NEXT: .LBB64_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB65_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB65_6
|
|
; GCN1-NEXT: .LBB65_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB65_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB65_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v1, s7, v3
|
|
; GCN1-NEXT: v_or_b32_e32 v0, s6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB65_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB65_2
|
|
; GCN1-NEXT: .LBB65_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB65_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB65_6
|
|
; GCN2-NEXT: .LBB65_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB65_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB65_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v1, s7, v3
|
|
; GCN2-NEXT: v_or_b32_e32 v0, s6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB65_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB65_2
|
|
; GCN2-NEXT: .LBB65_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB65_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB65_6
|
|
; GCN3-NEXT: .LBB65_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB65_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB65_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, s7, v3
|
|
; GCN3-NEXT: v_or_b32_e32 v0, s6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB65_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB65_2
|
|
; GCN3-NEXT: .LBB65_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB66_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB66_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_or_b32_e32 v5, s7, v7
|
|
; GCN1-NEXT: v_or_b32_e32 v4, s6, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB66_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB66_6
|
|
; GCN1-NEXT: .LBB66_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB66_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB66_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB66_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB66_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_or_b32_e32 v5, s7, v7
|
|
; GCN2-NEXT: v_or_b32_e32 v4, s6, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB66_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB66_6
|
|
; GCN2-NEXT: .LBB66_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB66_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB66_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB66_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB66_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_or_b32_e32 v5, s7, v7
|
|
; GCN3-NEXT: v_or_b32_e32 v4, s6, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB66_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB66_6
|
|
; GCN3-NEXT: .LBB66_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB66_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB66_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw or ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB67_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB67_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_or_b32_e32 v5, s7, v7
|
|
; GCN1-NEXT: v_or_b32_e32 v4, s6, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB67_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB67_6
|
|
; GCN1-NEXT: .LBB67_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB67_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB67_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB67_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB67_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_or_b32_e32 v5, s7, v7
|
|
; GCN2-NEXT: v_or_b32_e32 v4, s6, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB67_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB67_6
|
|
; GCN2-NEXT: .LBB67_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB67_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB67_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB67_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB67_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_or_b32_e32 v5, s7, v7
|
|
; GCN3-NEXT: v_or_b32_e32 v4, s6, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB67_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB67_6
|
|
; GCN3-NEXT: .LBB67_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB67_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB67_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw or ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB68_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB68_4
|
|
; GCN1-NEXT: .LBB68_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB68_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB68_2
|
|
; GCN1-NEXT: .LBB68_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB68_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB68_4
|
|
; GCN2-NEXT: .LBB68_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB68_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB68_2
|
|
; GCN2-NEXT: .LBB68_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB68_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB68_4
|
|
; GCN3-NEXT: .LBB68_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB68_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB68_2
|
|
; GCN3-NEXT: .LBB68_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB69_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB69_4
|
|
; GCN1-NEXT: .LBB69_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB69_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB69_2
|
|
; GCN1-NEXT: .LBB69_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB69_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB69_4
|
|
; GCN2-NEXT: .LBB69_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB69_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB69_2
|
|
; GCN2-NEXT: .LBB69_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB69_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB69_4
|
|
; GCN3-NEXT: .LBB69_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB69_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB69_2
|
|
; GCN3-NEXT: .LBB69_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw xor
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB70_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB70_6
|
|
; GCN1-NEXT: .LBB70_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB70_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB70_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB70_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB70_2
|
|
; GCN1-NEXT: .LBB70_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB70_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB70_6
|
|
; GCN2-NEXT: .LBB70_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB70_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB70_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB70_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB70_2
|
|
; GCN2-NEXT: .LBB70_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB70_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB70_6
|
|
; GCN3-NEXT: .LBB70_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB70_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB70_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v5, v7, v3
|
|
; GCN3-NEXT: v_xor_b32_e32 v4, v6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB70_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB70_2
|
|
; GCN3-NEXT: .LBB70_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB71_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB71_6
|
|
; GCN1-NEXT: .LBB71_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB71_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB71_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB71_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB71_2
|
|
; GCN1-NEXT: .LBB71_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB71_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB71_6
|
|
; GCN2-NEXT: .LBB71_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB71_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB71_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB71_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB71_2
|
|
; GCN2-NEXT: .LBB71_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB71_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB71_6
|
|
; GCN3-NEXT: .LBB71_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB71_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB71_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v5, v7, v3
|
|
; GCN3-NEXT: v_xor_b32_e32 v4, v6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB71_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB71_2
|
|
; GCN3-NEXT: .LBB71_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB72_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB72_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, v7, v3
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, v6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB72_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: .LBB72_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB72_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB72_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB72_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB72_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, v7, v3
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, v6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB72_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: .LBB72_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB72_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB72_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB72_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB72_6
|
|
; GCN3-NEXT: .LBB72_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB72_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB72_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_xor_b32_e32 v7, v9, v3
|
|
; GCN3-NEXT: v_xor_b32_e32 v6, v8, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB72_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB72_2
|
|
; GCN3-NEXT: .LBB72_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xor ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB73_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB73_6
|
|
; GCN1-NEXT: .LBB73_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB73_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB73_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_xor_b32_e32 v7, v9, v3
|
|
; GCN1-NEXT: v_xor_b32_e32 v6, v8, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB73_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB73_2
|
|
; GCN1-NEXT: .LBB73_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB73_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB73_6
|
|
; GCN2-NEXT: .LBB73_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB73_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB73_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_xor_b32_e32 v7, v9, v3
|
|
; GCN2-NEXT: v_xor_b32_e32 v6, v8, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB73_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB73_2
|
|
; GCN2-NEXT: .LBB73_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB73_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB73_6
|
|
; GCN3-NEXT: .LBB73_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB73_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB73_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_xor_b32_e32 v7, v9, v3
|
|
; GCN3-NEXT: v_xor_b32_e32 v6, v8, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB73_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB73_2
|
|
; GCN3-NEXT: .LBB73_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xor ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB74_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB74_6
|
|
; GCN1-NEXT: .LBB74_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB74_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB74_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v1, s7, v3
|
|
; GCN1-NEXT: v_xor_b32_e32 v0, s6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB74_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB74_2
|
|
; GCN1-NEXT: .LBB74_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB74_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB74_6
|
|
; GCN2-NEXT: .LBB74_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB74_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB74_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v1, s7, v3
|
|
; GCN2-NEXT: v_xor_b32_e32 v0, s6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB74_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB74_2
|
|
; GCN2-NEXT: .LBB74_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB74_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB74_6
|
|
; GCN3-NEXT: .LBB74_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB74_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB74_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v3
|
|
; GCN3-NEXT: v_xor_b32_e32 v0, s6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB74_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB74_2
|
|
; GCN3-NEXT: .LBB74_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB75_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB75_6
|
|
; GCN1-NEXT: .LBB75_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB75_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB75_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v1, s7, v3
|
|
; GCN1-NEXT: v_xor_b32_e32 v0, s6, v2
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB75_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB75_2
|
|
; GCN1-NEXT: .LBB75_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB75_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB75_6
|
|
; GCN2-NEXT: .LBB75_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB75_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB75_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v1, s7, v3
|
|
; GCN2-NEXT: v_xor_b32_e32 v0, s6, v2
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB75_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB75_2
|
|
; GCN2-NEXT: .LBB75_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB75_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB75_6
|
|
; GCN3-NEXT: .LBB75_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB75_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB75_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v3
|
|
; GCN3-NEXT: v_xor_b32_e32 v0, s6, v2
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB75_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB75_2
|
|
; GCN3-NEXT: .LBB75_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB76_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB76_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, s7, v7
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, s6, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB76_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB76_6
|
|
; GCN1-NEXT: .LBB76_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB76_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB76_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB76_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB76_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, s7, v7
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, s6, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB76_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB76_6
|
|
; GCN2-NEXT: .LBB76_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB76_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB76_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB76_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB76_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_xor_b32_e32 v5, s7, v7
|
|
; GCN3-NEXT: v_xor_b32_e32 v4, s6, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB76_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB76_6
|
|
; GCN3-NEXT: .LBB76_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB76_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB76_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xor ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB77_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB77_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, s7, v7
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, s6, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB77_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB77_6
|
|
; GCN1-NEXT: .LBB77_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB77_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB77_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB77_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB77_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, s7, v7
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, s6, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB77_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB77_6
|
|
; GCN2-NEXT: .LBB77_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB77_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB77_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB77_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB77_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_xor_b32_e32 v5, s7, v7
|
|
; GCN3-NEXT: v_xor_b32_e32 v4, s6, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB77_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB77_6
|
|
; GCN3-NEXT: .LBB77_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB77_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB77_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xor ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB78_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB78_4
|
|
; GCN1-NEXT: .LBB78_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB78_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB78_2
|
|
; GCN1-NEXT: .LBB78_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB78_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB78_4
|
|
; GCN2-NEXT: .LBB78_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB78_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB78_2
|
|
; GCN2-NEXT: .LBB78_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB78_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB78_4
|
|
; GCN3-NEXT: .LBB78_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB78_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB78_2
|
|
; GCN3-NEXT: .LBB78_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB79_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB79_4
|
|
; GCN1-NEXT: .LBB79_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB79_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB79_2
|
|
; GCN1-NEXT: .LBB79_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB79_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB79_4
|
|
; GCN2-NEXT: .LBB79_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB79_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB79_2
|
|
; GCN2-NEXT: .LBB79_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB79_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB79_4
|
|
; GCN3-NEXT: .LBB79_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB79_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB79_2
|
|
; GCN3-NEXT: .LBB79_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw max
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_max_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB80_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB80_6
|
|
; GCN1-NEXT: .LBB80_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB80_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB80_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB80_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB80_2
|
|
; GCN1-NEXT: .LBB80_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB80_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB80_6
|
|
; GCN2-NEXT: .LBB80_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB80_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB80_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB80_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB80_2
|
|
; GCN2-NEXT: .LBB80_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB80_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB80_6
|
|
; GCN3-NEXT: .LBB80_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB80_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB80_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB80_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB80_2
|
|
; GCN3-NEXT: .LBB80_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_max_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB81_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB81_6
|
|
; GCN1-NEXT: .LBB81_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB81_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB81_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB81_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB81_2
|
|
; GCN1-NEXT: .LBB81_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB81_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB81_6
|
|
; GCN2-NEXT: .LBB81_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB81_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB81_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB81_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB81_2
|
|
; GCN2-NEXT: .LBB81_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB81_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB81_6
|
|
; GCN3-NEXT: .LBB81_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB81_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB81_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB81_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB81_2
|
|
; GCN3-NEXT: .LBB81_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_max_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB82_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB82_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB82_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: .LBB82_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB82_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB82_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB82_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB82_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB82_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: .LBB82_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB82_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB82_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB82_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB82_6
|
|
; GCN3-NEXT: .LBB82_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB82_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB82_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB82_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB82_2
|
|
; GCN3-NEXT: .LBB82_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw max ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_max_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB83_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB83_6
|
|
; GCN1-NEXT: .LBB83_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB83_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB83_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB83_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB83_2
|
|
; GCN1-NEXT: .LBB83_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB83_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB83_6
|
|
; GCN2-NEXT: .LBB83_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB83_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB83_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB83_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB83_2
|
|
; GCN2-NEXT: .LBB83_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB83_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB83_6
|
|
; GCN3-NEXT: .LBB83_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB83_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB83_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB83_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB83_2
|
|
; GCN3-NEXT: .LBB83_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw max ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB84_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB84_6
|
|
; GCN1-NEXT: .LBB84_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB84_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB84_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB84_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB84_2
|
|
; GCN1-NEXT: .LBB84_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB84_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB84_6
|
|
; GCN2-NEXT: .LBB84_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB84_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB84_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB84_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB84_2
|
|
; GCN2-NEXT: .LBB84_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB84_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB84_6
|
|
; GCN3-NEXT: .LBB84_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB84_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB84_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB84_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB84_2
|
|
; GCN3-NEXT: .LBB84_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB85_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB85_6
|
|
; GCN1-NEXT: .LBB85_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB85_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB85_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB85_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB85_2
|
|
; GCN1-NEXT: .LBB85_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB85_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB85_6
|
|
; GCN2-NEXT: .LBB85_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB85_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB85_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB85_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB85_2
|
|
; GCN2-NEXT: .LBB85_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB85_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB85_6
|
|
; GCN3-NEXT: .LBB85_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB85_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB85_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB85_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB85_2
|
|
; GCN3-NEXT: .LBB85_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB86_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB86_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB86_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB86_6
|
|
; GCN1-NEXT: .LBB86_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB86_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB86_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB86_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB86_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB86_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB86_6
|
|
; GCN2-NEXT: .LBB86_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB86_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB86_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB86_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB86_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB86_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB86_6
|
|
; GCN3-NEXT: .LBB86_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB86_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB86_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw max ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB87_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB87_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB87_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB87_6
|
|
; GCN1-NEXT: .LBB87_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB87_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB87_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB87_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB87_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB87_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB87_6
|
|
; GCN2-NEXT: .LBB87_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB87_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB87_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB87_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB87_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB87_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB87_6
|
|
; GCN3-NEXT: .LBB87_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB87_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB87_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw max ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_max_i64_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s14, -1
|
|
; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f
|
|
; GCN1-NEXT: s_mov_b32 s15, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN1-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
|
|
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB88_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB88_6
|
|
; GCN1-NEXT: .LBB88_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_endpgm
|
|
; GCN1-NEXT: .LBB88_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB88_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB88_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_branch .LBB88_2
|
|
; GCN1-NEXT: .LBB88_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_max_i64_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
|
|
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB88_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB88_6
|
|
; GCN2-NEXT: .LBB88_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_endpgm
|
|
; GCN2-NEXT: .LBB88_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB88_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB88_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_branch .LBB88_2
|
|
; GCN2-NEXT: .LBB88_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_max_i64_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
|
|
; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN3-NEXT: s_mov_b32 s14, -1
|
|
; GCN3-NEXT: s_mov_b32 s15, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN3-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN3-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s5
|
|
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB88_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB88_6
|
|
; GCN3-NEXT: .LBB88_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_endpgm
|
|
; GCN3-NEXT: .LBB88_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN3-NEXT: .LBB88_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB88_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_branch .LBB88_2
|
|
; GCN3-NEXT: .LBB88_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s2
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_max_i64_ret_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s18, -1
|
|
; GCN1-NEXT: s_mov_b32 s19, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41
|
|
; GCN1-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN1-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB89_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: .LBB89_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB89_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_branch .LBB89_6
|
|
; GCN1-NEXT: .LBB89_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB89_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: .LBB89_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_max_i64_ret_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN2-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB89_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: .LBB89_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB89_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_branch .LBB89_6
|
|
; GCN2-NEXT: .LBB89_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB89_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: .LBB89_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_max_i64_ret_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_mov_b32 s18, -1
|
|
; GCN3-NEXT: s_mov_b32 s19, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN3-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN3-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
|
|
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB89_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN3-NEXT: .LBB89_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB89_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_branch .LBB89_6
|
|
; GCN3-NEXT: .LBB89_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB89_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s12
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s13
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB89_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_max_i64_addr64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s14, -1
|
|
; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f
|
|
; GCN1-NEXT: s_mov_b32 s15, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN1-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
|
|
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB90_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB90_6
|
|
; GCN1-NEXT: .LBB90_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_endpgm
|
|
; GCN1-NEXT: .LBB90_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB90_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB90_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_branch .LBB90_2
|
|
; GCN1-NEXT: .LBB90_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_max_i64_addr64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
|
|
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB90_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB90_6
|
|
; GCN2-NEXT: .LBB90_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_endpgm
|
|
; GCN2-NEXT: .LBB90_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB90_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB90_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_branch .LBB90_2
|
|
; GCN2-NEXT: .LBB90_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_max_i64_addr64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
|
|
; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN3-NEXT: s_mov_b32 s14, -1
|
|
; GCN3-NEXT: s_mov_b32 s15, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN3-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s5
|
|
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB90_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB90_6
|
|
; GCN3-NEXT: .LBB90_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_endpgm
|
|
; GCN3-NEXT: .LBB90_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN3-NEXT: .LBB90_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB90_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_branch .LBB90_2
|
|
; GCN3-NEXT: .LBB90_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s2
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_max_i64_ret_addr64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s18, -1
|
|
; GCN1-NEXT: s_mov_b32 s19, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41
|
|
; GCN1-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN1-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB91_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: .LBB91_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB91_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_branch .LBB91_6
|
|
; GCN1-NEXT: .LBB91_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB91_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: .LBB91_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_max_i64_ret_addr64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN2-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB91_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: .LBB91_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB91_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_branch .LBB91_6
|
|
; GCN2-NEXT: .LBB91_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB91_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: .LBB91_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_max_i64_ret_addr64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_mov_b32 s18, -1
|
|
; GCN3-NEXT: s_mov_b32 s19, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN3-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
|
|
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB91_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN3-NEXT: .LBB91_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB91_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_branch .LBB91_6
|
|
; GCN3-NEXT: .LBB91_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB91_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s12
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s13
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB91_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB92_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB92_4
|
|
; GCN1-NEXT: .LBB92_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB92_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB92_2
|
|
; GCN1-NEXT: .LBB92_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB92_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB92_4
|
|
; GCN2-NEXT: .LBB92_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB92_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB92_2
|
|
; GCN2-NEXT: .LBB92_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB92_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB92_4
|
|
; GCN3-NEXT: .LBB92_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB92_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB92_2
|
|
; GCN3-NEXT: .LBB92_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB93_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB93_4
|
|
; GCN1-NEXT: .LBB93_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB93_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB93_2
|
|
; GCN1-NEXT: .LBB93_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB93_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB93_4
|
|
; GCN2-NEXT: .LBB93_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB93_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB93_2
|
|
; GCN2-NEXT: .LBB93_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB93_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB93_4
|
|
; GCN3-NEXT: .LBB93_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB93_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB93_2
|
|
; GCN3-NEXT: .LBB93_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw umax
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_umax_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB94_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB94_6
|
|
; GCN1-NEXT: .LBB94_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB94_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB94_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB94_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB94_2
|
|
; GCN1-NEXT: .LBB94_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB94_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB94_6
|
|
; GCN2-NEXT: .LBB94_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB94_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB94_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB94_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB94_2
|
|
; GCN2-NEXT: .LBB94_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB94_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB94_6
|
|
; GCN3-NEXT: .LBB94_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB94_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB94_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB94_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB94_2
|
|
; GCN3-NEXT: .LBB94_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_umax_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB95_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB95_6
|
|
; GCN1-NEXT: .LBB95_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB95_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB95_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB95_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB95_2
|
|
; GCN1-NEXT: .LBB95_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB95_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB95_6
|
|
; GCN2-NEXT: .LBB95_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB95_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB95_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB95_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB95_2
|
|
; GCN2-NEXT: .LBB95_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB95_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB95_6
|
|
; GCN3-NEXT: .LBB95_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB95_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB95_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB95_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB95_2
|
|
; GCN3-NEXT: .LBB95_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_umax_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB96_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB96_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB96_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: .LBB96_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB96_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB96_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB96_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB96_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB96_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: .LBB96_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB96_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB96_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB96_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB96_6
|
|
; GCN3-NEXT: .LBB96_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB96_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB96_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB96_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB96_2
|
|
; GCN3-NEXT: .LBB96_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw umax ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_umax_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB97_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB97_6
|
|
; GCN1-NEXT: .LBB97_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB97_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB97_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB97_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB97_2
|
|
; GCN1-NEXT: .LBB97_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB97_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB97_6
|
|
; GCN2-NEXT: .LBB97_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB97_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB97_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB97_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB97_2
|
|
; GCN2-NEXT: .LBB97_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB97_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB97_6
|
|
; GCN3-NEXT: .LBB97_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB97_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB97_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB97_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB97_2
|
|
; GCN3-NEXT: .LBB97_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umax ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB98_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB98_6
|
|
; GCN1-NEXT: .LBB98_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB98_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB98_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB98_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB98_2
|
|
; GCN1-NEXT: .LBB98_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB98_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB98_6
|
|
; GCN2-NEXT: .LBB98_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB98_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB98_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB98_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB98_2
|
|
; GCN2-NEXT: .LBB98_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB98_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB98_6
|
|
; GCN3-NEXT: .LBB98_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB98_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB98_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB98_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB98_2
|
|
; GCN3-NEXT: .LBB98_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB99_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB99_6
|
|
; GCN1-NEXT: .LBB99_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB99_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB99_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB99_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB99_2
|
|
; GCN1-NEXT: .LBB99_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB99_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB99_6
|
|
; GCN2-NEXT: .LBB99_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB99_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB99_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB99_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB99_2
|
|
; GCN2-NEXT: .LBB99_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB99_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB99_6
|
|
; GCN3-NEXT: .LBB99_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB99_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB99_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB99_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB99_2
|
|
; GCN3-NEXT: .LBB99_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB100_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB100_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB100_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB100_6
|
|
; GCN1-NEXT: .LBB100_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB100_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB100_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB100_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB100_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB100_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB100_6
|
|
; GCN2-NEXT: .LBB100_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB100_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB100_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB100_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB100_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB100_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB100_6
|
|
; GCN3-NEXT: .LBB100_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB100_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB100_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw umax ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB101_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB101_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB101_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB101_6
|
|
; GCN1-NEXT: .LBB101_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB101_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB101_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB101_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB101_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB101_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB101_6
|
|
; GCN2-NEXT: .LBB101_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB101_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB101_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB101_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB101_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB101_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB101_6
|
|
; GCN3-NEXT: .LBB101_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB101_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB101_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umax ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_umax_i64_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s14, -1
|
|
; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f
|
|
; GCN1-NEXT: s_mov_b32 s15, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN1-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
|
|
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB102_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB102_6
|
|
; GCN1-NEXT: .LBB102_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_endpgm
|
|
; GCN1-NEXT: .LBB102_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB102_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB102_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_branch .LBB102_2
|
|
; GCN1-NEXT: .LBB102_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_umax_i64_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
|
|
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB102_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB102_6
|
|
; GCN2-NEXT: .LBB102_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_endpgm
|
|
; GCN2-NEXT: .LBB102_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB102_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB102_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_branch .LBB102_2
|
|
; GCN2-NEXT: .LBB102_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_umax_i64_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
|
|
; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN3-NEXT: s_mov_b32 s14, -1
|
|
; GCN3-NEXT: s_mov_b32 s15, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN3-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN3-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s5
|
|
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB102_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB102_6
|
|
; GCN3-NEXT: .LBB102_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_endpgm
|
|
; GCN3-NEXT: .LBB102_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN3-NEXT: .LBB102_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB102_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_branch .LBB102_2
|
|
; GCN3-NEXT: .LBB102_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s2
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_umax_i64_ret_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s18, -1
|
|
; GCN1-NEXT: s_mov_b32 s19, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41
|
|
; GCN1-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN1-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB103_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: .LBB103_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB103_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_branch .LBB103_6
|
|
; GCN1-NEXT: .LBB103_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB103_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: .LBB103_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN2-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB103_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: .LBB103_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB103_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_branch .LBB103_6
|
|
; GCN2-NEXT: .LBB103_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB103_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: .LBB103_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_umax_i64_ret_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_mov_b32 s18, -1
|
|
; GCN3-NEXT: s_mov_b32 s19, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN3-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN3-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
|
|
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB103_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN3-NEXT: .LBB103_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB103_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_branch .LBB103_6
|
|
; GCN3-NEXT: .LBB103_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB103_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s12
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s13
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB103_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_umax_i64_ret_addr64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s18, -1
|
|
; GCN1-NEXT: s_mov_b32 s19, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41
|
|
; GCN1-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN1-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB104_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: .LBB104_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB104_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_branch .LBB104_6
|
|
; GCN1-NEXT: .LBB104_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB104_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: .LBB104_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_umax_i64_ret_addr64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN2-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB104_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: .LBB104_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB104_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_branch .LBB104_6
|
|
; GCN2-NEXT: .LBB104_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB104_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: .LBB104_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_umax_i64_ret_addr64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_mov_b32 s18, -1
|
|
; GCN3-NEXT: s_mov_b32 s19, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN3-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
|
|
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB104_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN3-NEXT: .LBB104_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB104_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_branch .LBB104_6
|
|
; GCN3-NEXT: .LBB104_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB104_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s12
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s13
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB104_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB105_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB105_4
|
|
; GCN1-NEXT: .LBB105_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB105_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB105_2
|
|
; GCN1-NEXT: .LBB105_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB105_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB105_4
|
|
; GCN2-NEXT: .LBB105_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB105_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB105_2
|
|
; GCN2-NEXT: .LBB105_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB105_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB105_4
|
|
; GCN3-NEXT: .LBB105_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB105_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB105_2
|
|
; GCN3-NEXT: .LBB105_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB106_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB106_4
|
|
; GCN1-NEXT: .LBB106_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB106_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB106_2
|
|
; GCN1-NEXT: .LBB106_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB106_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB106_4
|
|
; GCN2-NEXT: .LBB106_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB106_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB106_2
|
|
; GCN2-NEXT: .LBB106_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB106_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB106_4
|
|
; GCN3-NEXT: .LBB106_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB106_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB106_2
|
|
; GCN3-NEXT: .LBB106_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw umin
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_umin_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB107_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB107_6
|
|
; GCN1-NEXT: .LBB107_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB107_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB107_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB107_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB107_2
|
|
; GCN1-NEXT: .LBB107_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB107_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB107_6
|
|
; GCN2-NEXT: .LBB107_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB107_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB107_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB107_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB107_2
|
|
; GCN2-NEXT: .LBB107_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB107_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB107_6
|
|
; GCN3-NEXT: .LBB107_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB107_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB107_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB107_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB107_2
|
|
; GCN3-NEXT: .LBB107_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_umin_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB108_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB108_6
|
|
; GCN1-NEXT: .LBB108_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB108_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB108_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB108_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB108_2
|
|
; GCN1-NEXT: .LBB108_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB108_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB108_6
|
|
; GCN2-NEXT: .LBB108_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB108_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB108_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB108_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB108_2
|
|
; GCN2-NEXT: .LBB108_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB108_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB108_6
|
|
; GCN3-NEXT: .LBB108_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB108_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB108_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB108_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB108_2
|
|
; GCN3-NEXT: .LBB108_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_umin_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB109_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB109_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB109_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: .LBB109_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB109_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB109_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB109_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB109_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB109_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: .LBB109_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB109_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB109_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB109_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB109_6
|
|
; GCN3-NEXT: .LBB109_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB109_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB109_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB109_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB109_2
|
|
; GCN3-NEXT: .LBB109_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw umin ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_umin_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB110_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB110_6
|
|
; GCN1-NEXT: .LBB110_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB110_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB110_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB110_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB110_2
|
|
; GCN1-NEXT: .LBB110_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB110_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB110_6
|
|
; GCN2-NEXT: .LBB110_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB110_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB110_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB110_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB110_2
|
|
; GCN2-NEXT: .LBB110_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB110_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB110_6
|
|
; GCN3-NEXT: .LBB110_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB110_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB110_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB110_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB110_2
|
|
; GCN3-NEXT: .LBB110_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umin ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB111_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB111_6
|
|
; GCN1-NEXT: .LBB111_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB111_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB111_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB111_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB111_2
|
|
; GCN1-NEXT: .LBB111_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB111_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB111_6
|
|
; GCN2-NEXT: .LBB111_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB111_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB111_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB111_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB111_2
|
|
; GCN2-NEXT: .LBB111_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB111_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB111_6
|
|
; GCN3-NEXT: .LBB111_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB111_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB111_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB111_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB111_2
|
|
; GCN3-NEXT: .LBB111_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB112_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB112_6
|
|
; GCN1-NEXT: .LBB112_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB112_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB112_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB112_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB112_2
|
|
; GCN1-NEXT: .LBB112_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB112_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB112_6
|
|
; GCN2-NEXT: .LBB112_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB112_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB112_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB112_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB112_2
|
|
; GCN2-NEXT: .LBB112_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB112_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB112_6
|
|
; GCN3-NEXT: .LBB112_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB112_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB112_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB112_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB112_2
|
|
; GCN3-NEXT: .LBB112_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB113_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB113_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB113_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB113_6
|
|
; GCN1-NEXT: .LBB113_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB113_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB113_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB113_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB113_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB113_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB113_6
|
|
; GCN2-NEXT: .LBB113_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB113_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB113_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB113_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB113_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB113_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB113_6
|
|
; GCN3-NEXT: .LBB113_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB113_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB113_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw umin ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB114_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB114_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB114_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB114_6
|
|
; GCN1-NEXT: .LBB114_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB114_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB114_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB114_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB114_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB114_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB114_6
|
|
; GCN2-NEXT: .LBB114_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB114_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB114_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB114_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB114_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB114_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB114_6
|
|
; GCN3-NEXT: .LBB114_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB114_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB114_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umin ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB115_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB115_4
|
|
; GCN1-NEXT: .LBB115_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB115_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB115_2
|
|
; GCN1-NEXT: .LBB115_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB115_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB115_4
|
|
; GCN2-NEXT: .LBB115_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB115_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB115_2
|
|
; GCN2-NEXT: .LBB115_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB115_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB115_4
|
|
; GCN3-NEXT: .LBB115_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB115_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB115_2
|
|
; GCN3-NEXT: .LBB115_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB116_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB116_4
|
|
; GCN1-NEXT: .LBB116_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB116_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB116_2
|
|
; GCN1-NEXT: .LBB116_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB116_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB116_4
|
|
; GCN2-NEXT: .LBB116_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB116_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB116_2
|
|
; GCN2-NEXT: .LBB116_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB116_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB116_4
|
|
; GCN3-NEXT: .LBB116_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB116_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB116_2
|
|
; GCN3-NEXT: .LBB116_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw min
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_min_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB117_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB117_6
|
|
; GCN1-NEXT: .LBB117_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB117_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB117_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB117_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB117_2
|
|
; GCN1-NEXT: .LBB117_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB117_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB117_6
|
|
; GCN2-NEXT: .LBB117_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB117_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB117_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB117_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB117_2
|
|
; GCN2-NEXT: .LBB117_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB117_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB117_6
|
|
; GCN3-NEXT: .LBB117_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB117_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB117_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB117_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB117_2
|
|
; GCN3-NEXT: .LBB117_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_min_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB118_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB118_6
|
|
; GCN1-NEXT: .LBB118_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB118_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB118_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB118_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB118_2
|
|
; GCN1-NEXT: .LBB118_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB118_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB118_6
|
|
; GCN2-NEXT: .LBB118_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB118_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB118_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB118_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB118_2
|
|
; GCN2-NEXT: .LBB118_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB118_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB118_6
|
|
; GCN3-NEXT: .LBB118_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB118_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB118_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB118_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB118_2
|
|
; GCN3-NEXT: .LBB118_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_min_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB119_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB119_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB119_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: .LBB119_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB119_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB119_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB119_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB119_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB119_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: .LBB119_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB119_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB119_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB119_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB119_6
|
|
; GCN3-NEXT: .LBB119_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB119_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB119_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB119_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB119_2
|
|
; GCN3-NEXT: .LBB119_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw min ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_min_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB120_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB120_6
|
|
; GCN1-NEXT: .LBB120_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB120_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB120_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB120_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB120_2
|
|
; GCN1-NEXT: .LBB120_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB120_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB120_6
|
|
; GCN2-NEXT: .LBB120_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB120_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB120_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB120_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB120_2
|
|
; GCN2-NEXT: .LBB120_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB120_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB120_6
|
|
; GCN3-NEXT: .LBB120_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB120_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB120_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB120_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB120_2
|
|
; GCN3-NEXT: .LBB120_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw min ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB121_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB121_6
|
|
; GCN1-NEXT: .LBB121_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB121_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB121_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB121_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB121_2
|
|
; GCN1-NEXT: .LBB121_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB121_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB121_6
|
|
; GCN2-NEXT: .LBB121_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB121_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB121_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB121_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB121_2
|
|
; GCN2-NEXT: .LBB121_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB121_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB121_6
|
|
; GCN3-NEXT: .LBB121_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB121_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB121_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB121_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB121_2
|
|
; GCN3-NEXT: .LBB121_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB122_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB122_6
|
|
; GCN1-NEXT: .LBB122_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB122_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB122_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB122_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB122_2
|
|
; GCN1-NEXT: .LBB122_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB122_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB122_6
|
|
; GCN2-NEXT: .LBB122_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB122_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB122_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB122_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB122_2
|
|
; GCN2-NEXT: .LBB122_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB122_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB122_6
|
|
; GCN3-NEXT: .LBB122_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB122_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB122_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB122_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB122_2
|
|
; GCN3-NEXT: .LBB122_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB123_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB123_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB123_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB123_6
|
|
; GCN1-NEXT: .LBB123_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB123_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB123_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB123_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB123_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB123_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB123_6
|
|
; GCN2-NEXT: .LBB123_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB123_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB123_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB123_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB123_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB123_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB123_6
|
|
; GCN3-NEXT: .LBB123_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB123_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB123_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw min ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB124_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB124_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB124_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB124_6
|
|
; GCN1-NEXT: .LBB124_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB124_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB124_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB124_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB124_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB124_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB124_6
|
|
; GCN2-NEXT: .LBB124_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB124_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB124_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB124_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB124_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB124_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB124_6
|
|
; GCN3-NEXT: .LBB124_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB124_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB124_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw min ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_min_i64_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s14, -1
|
|
; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f
|
|
; GCN1-NEXT: s_mov_b32 s15, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN1-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
|
|
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB125_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB125_6
|
|
; GCN1-NEXT: .LBB125_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_endpgm
|
|
; GCN1-NEXT: .LBB125_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB125_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB125_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_branch .LBB125_2
|
|
; GCN1-NEXT: .LBB125_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_min_i64_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
|
|
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB125_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB125_6
|
|
; GCN2-NEXT: .LBB125_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_endpgm
|
|
; GCN2-NEXT: .LBB125_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB125_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB125_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_branch .LBB125_2
|
|
; GCN2-NEXT: .LBB125_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_min_i64_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
|
|
; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN3-NEXT: s_mov_b32 s14, -1
|
|
; GCN3-NEXT: s_mov_b32 s15, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN3-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN3-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s5
|
|
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB125_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow6
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB125_6
|
|
; GCN3-NEXT: .LBB125_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_endpgm
|
|
; GCN3-NEXT: .LBB125_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN3-NEXT: .LBB125_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB125_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_branch .LBB125_2
|
|
; GCN3-NEXT: .LBB125_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s2
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_min_i64_ret_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s18, -1
|
|
; GCN1-NEXT: s_mov_b32 s19, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41
|
|
; GCN1-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN1-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB126_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: .LBB126_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB126_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_branch .LBB126_6
|
|
; GCN1-NEXT: .LBB126_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB126_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: .LBB126_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_min_i64_ret_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN2-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB126_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: .LBB126_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB126_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_branch .LBB126_6
|
|
; GCN2-NEXT: .LBB126_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB126_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: .LBB126_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_min_i64_ret_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_mov_b32 s18, -1
|
|
; GCN3-NEXT: s_mov_b32 s19, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN3-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN3-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN3-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
|
|
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB126_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN3-NEXT: .LBB126_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB126_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_branch .LBB126_6
|
|
; GCN3-NEXT: .LBB126_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB126_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s12
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s13
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB126_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: atomic_min_i64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d
|
|
; GCN1-NEXT: s_mov_b32 s14, -1
|
|
; GCN1-NEXT: s_mov_b32 s15, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN1-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
|
|
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB127_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow5
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB127_6
|
|
; GCN1-NEXT: .LBB127_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_endpgm
|
|
; GCN1-NEXT: .LBB127_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB127_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB127_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_branch .LBB127_2
|
|
; GCN1-NEXT: .LBB127_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_min_i64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf4
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
|
|
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB127_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow5
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB127_6
|
|
; GCN2-NEXT: .LBB127_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_endpgm
|
|
; GCN2-NEXT: .LBB127_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB127_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB127_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_branch .LBB127_2
|
|
; GCN2-NEXT: .LBB127_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s3
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_min_i64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
|
|
; GCN3-NEXT: s_mov_b32 s14, -1
|
|
; GCN3-NEXT: s_mov_b32 s15, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s12, s12, s11
|
|
; GCN3-NEXT: s_addc_u32 s13, s13, 0
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s5
|
|
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB127_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow5
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB127_6
|
|
; GCN3-NEXT: .LBB127_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_endpgm
|
|
; GCN3-NEXT: .LBB127_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN3-NEXT: .LBB127_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB127_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_branch .LBB127_2
|
|
; GCN3-NEXT: .LBB127_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s2
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%tmp0 = atomicrmw min ptr %out, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_min_i64_ret_addr64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN1-NEXT: s_mov_b32 s18, -1
|
|
; GCN1-NEXT: s_mov_b32 s19, 0xe8f000
|
|
; GCN1-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
|
|
; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41
|
|
; GCN1-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN1-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB128_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: .LBB128_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB128_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN1-NEXT: s_branch .LBB128_6
|
|
; GCN1-NEXT: .LBB128_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB128_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
|
|
; GCN1-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN1-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen
|
|
; GCN1-NEXT: .LBB128_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_min_i64_ret_addr64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
|
|
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
|
|
; GCN2-NEXT: s_mov_b32 s90, -1
|
|
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
|
|
; GCN2-NEXT: s_add_u32 s88, s88, s11
|
|
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
|
|
; GCN2-NEXT: s_addc_u32 s89, s89, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN2-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
|
|
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB128_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: .LBB128_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB128_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN2-NEXT: s_branch .LBB128_6
|
|
; GCN2-NEXT: .LBB128_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB128_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN2-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN2-NEXT: s_add_i32 s0, s0, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s0
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen
|
|
; GCN2-NEXT: .LBB128_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_min_i64_ret_addr64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
|
|
; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
|
|
; GCN3-NEXT: s_mov_b32 s18, -1
|
|
; GCN3-NEXT: s_mov_b32 s19, 0xe00000
|
|
; GCN3-NEXT: s_add_u32 s16, s16, s11
|
|
; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
|
|
; GCN3-NEXT: s_addc_u32 s17, s17, 0
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s8, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s9, s1
|
|
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
|
|
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB128_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[2:3], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s13
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s12
|
|
; GCN3-NEXT: .LBB128_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB128_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[2:3]
|
|
; GCN3-NEXT: s_branch .LBB128_6
|
|
; GCN3-NEXT: .LBB128_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB128_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0
|
|
; GCN3-NEXT: s_cselect_b32 s0, s0, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s0
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s12
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s13
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB128_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s10
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s11
|
|
; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB129_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB129_4
|
|
; GCN1-NEXT: .LBB129_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB129_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB129_2
|
|
; GCN1-NEXT: .LBB129_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB129_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB129_4
|
|
; GCN2-NEXT: .LBB129_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB129_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB129_2
|
|
; GCN2-NEXT: .LBB129_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB129_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB129_4
|
|
; GCN3-NEXT: .LBB129_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB129_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB129_2
|
|
; GCN3-NEXT: .LBB129_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB130_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB130_4
|
|
; GCN1-NEXT: .LBB130_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB130_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB130_2
|
|
; GCN1-NEXT: .LBB130_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB130_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB130_4
|
|
; GCN2-NEXT: .LBB130_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB130_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB130_2
|
|
; GCN2-NEXT: .LBB130_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB130_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB130_4
|
|
; GCN3-NEXT: .LBB130_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB130_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB130_2
|
|
; GCN3-NEXT: .LBB130_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw uinc_wrap
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB131_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB131_6
|
|
; GCN1-NEXT: .LBB131_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB131_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB131_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v6
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB131_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB131_2
|
|
; GCN1-NEXT: .LBB131_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB131_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB131_6
|
|
; GCN2-NEXT: .LBB131_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB131_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB131_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v6
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB131_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB131_2
|
|
; GCN2-NEXT: .LBB131_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB131_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB131_6
|
|
; GCN3-NEXT: .LBB131_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB131_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB131_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 1, v6
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB131_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB131_2
|
|
; GCN3-NEXT: .LBB131_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB132_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB132_6
|
|
; GCN1-NEXT: .LBB132_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB132_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB132_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v6
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB132_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB132_2
|
|
; GCN1-NEXT: .LBB132_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB132_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB132_6
|
|
; GCN2-NEXT: .LBB132_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB132_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB132_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v6
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB132_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB132_2
|
|
; GCN2-NEXT: .LBB132_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB132_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB132_6
|
|
; GCN3-NEXT: .LBB132_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB132_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB132_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 1, v6
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB132_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB132_2
|
|
; GCN3-NEXT: .LBB132_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB133_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB133_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v6
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB133_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: .LBB133_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB133_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v4
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB133_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB133_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB133_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v6
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB133_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: .LBB133_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB133_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v4
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB133_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB133_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB133_6
|
|
; GCN3-NEXT: .LBB133_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB133_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB133_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v8
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB133_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB133_2
|
|
; GCN3-NEXT: .LBB133_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB134_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB134_6
|
|
; GCN1-NEXT: .LBB134_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB134_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB134_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v8
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v9, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB134_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB134_2
|
|
; GCN1-NEXT: .LBB134_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc
|
|
; GCN1-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB134_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB134_6
|
|
; GCN2-NEXT: .LBB134_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB134_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB134_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v8
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v9, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB134_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB134_2
|
|
; GCN2-NEXT: .LBB134_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc
|
|
; GCN2-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB134_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB134_6
|
|
; GCN3-NEXT: .LBB134_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB134_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB134_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v8
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB134_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB134_2
|
|
; GCN3-NEXT: .LBB134_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB135_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB135_6
|
|
; GCN1-NEXT: .LBB135_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB135_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB135_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v2
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB135_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB135_2
|
|
; GCN1-NEXT: .LBB135_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB135_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB135_6
|
|
; GCN2-NEXT: .LBB135_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB135_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB135_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v2
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB135_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB135_2
|
|
; GCN2-NEXT: .LBB135_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB135_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB135_6
|
|
; GCN3-NEXT: .LBB135_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB135_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB135_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB135_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB135_2
|
|
; GCN3-NEXT: .LBB135_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB136_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB136_6
|
|
; GCN1-NEXT: .LBB136_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB136_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB136_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v2
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB136_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB136_2
|
|
; GCN1-NEXT: .LBB136_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB136_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB136_6
|
|
; GCN2-NEXT: .LBB136_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB136_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB136_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v2
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB136_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB136_2
|
|
; GCN2-NEXT: .LBB136_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB136_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB136_6
|
|
; GCN3-NEXT: .LBB136_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB136_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB136_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB136_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB136_2
|
|
; GCN3-NEXT: .LBB136_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB137_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB137_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v6
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB137_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB137_6
|
|
; GCN1-NEXT: .LBB137_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB137_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB137_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB137_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB137_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v6
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB137_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB137_6
|
|
; GCN2-NEXT: .LBB137_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB137_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB137_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB137_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB137_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v6
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB137_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB137_6
|
|
; GCN3-NEXT: .LBB137_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB137_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB137_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB138_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB138_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v6
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB138_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB138_6
|
|
; GCN1-NEXT: .LBB138_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB138_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB138_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB138_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB138_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v6
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB138_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB138_6
|
|
; GCN2-NEXT: .LBB138_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB138_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB138_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB138_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB138_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v6
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB138_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB138_6
|
|
; GCN3-NEXT: .LBB138_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB138_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB138_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB139_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB139_4
|
|
; GCN1-NEXT: .LBB139_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB139_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB139_2
|
|
; GCN1-NEXT: .LBB139_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB139_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB139_4
|
|
; GCN2-NEXT: .LBB139_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB139_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB139_2
|
|
; GCN2-NEXT: .LBB139_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB139_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB139_4
|
|
; GCN3-NEXT: .LBB139_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB139_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB139_2
|
|
; GCN3-NEXT: .LBB139_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB140_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB140_4
|
|
; GCN1-NEXT: .LBB140_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB140_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB140_2
|
|
; GCN1-NEXT: .LBB140_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc
|
|
; GCN1-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB140_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB140_4
|
|
; GCN2-NEXT: .LBB140_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB140_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB140_2
|
|
; GCN2-NEXT: .LBB140_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc
|
|
; GCN2-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB140_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB140_4
|
|
; GCN3-NEXT: .LBB140_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB140_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB140_2
|
|
; GCN3-NEXT: .LBB140_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw udec_wrap
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB141_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB141_6
|
|
; GCN1-NEXT: .LBB141_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB141_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN1-NEXT: .LBB141_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v4, s[6:7], -1, v6
|
|
; GCN1-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB141_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB141_2
|
|
; GCN1-NEXT: .LBB141_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB141_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB141_6
|
|
; GCN2-NEXT: .LBB141_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB141_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN2-NEXT: .LBB141_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v4, s[6:7], -1, v6
|
|
; GCN2-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB141_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB141_2
|
|
; GCN2-NEXT: .LBB141_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB141_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB141_6
|
|
; GCN3-NEXT: .LBB141_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB141_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN3-NEXT: .LBB141_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v6
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB141_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB141_2
|
|
; GCN3-NEXT: .LBB141_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB142_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB142_6
|
|
; GCN1-NEXT: .LBB142_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB142_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN1-NEXT: .LBB142_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v4, s[6:7], -1, v6
|
|
; GCN1-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB142_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB142_2
|
|
; GCN1-NEXT: .LBB142_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB142_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB142_6
|
|
; GCN2-NEXT: .LBB142_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB142_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN2-NEXT: .LBB142_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v4, s[6:7], -1, v6
|
|
; GCN2-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB142_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB142_2
|
|
; GCN2-NEXT: .LBB142_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB142_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB142_6
|
|
; GCN3-NEXT: .LBB142_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB142_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN3-NEXT: .LBB142_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v6
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB142_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB142_2
|
|
; GCN3-NEXT: .LBB142_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB143_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN1-NEXT: .LBB143_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v4, s[6:7], -1, v6
|
|
; GCN1-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB143_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: .LBB143_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB143_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v4
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[4:5], v[2:3]
|
|
; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v5, s[6:7]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB143_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB143_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN2-NEXT: .LBB143_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v4, s[6:7], -1, v6
|
|
; GCN2-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB143_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: .LBB143_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB143_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v4
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[4:5], v[2:3]
|
|
; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v5, s[6:7]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB143_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB143_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB143_6
|
|
; GCN3-NEXT: .LBB143_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB143_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN3-NEXT: .LBB143_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v8
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v9, s[6:7]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB143_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB143_2
|
|
; GCN3-NEXT: .LBB143_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB144_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB144_6
|
|
; GCN1-NEXT: .LBB144_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB144_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN1-NEXT: .LBB144_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v8
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB144_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB144_2
|
|
; GCN1-NEXT: .LBB144_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB144_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB144_6
|
|
; GCN2-NEXT: .LBB144_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB144_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN2-NEXT: .LBB144_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v8
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB144_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB144_2
|
|
; GCN2-NEXT: .LBB144_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB144_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB144_6
|
|
; GCN3-NEXT: .LBB144_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB144_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[10:11], 0
|
|
; GCN3-NEXT: .LBB144_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v8
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v9, s[6:7]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[10:11]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB144_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[10:11]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[8:9]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB144_2
|
|
; GCN3-NEXT: .LBB144_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB145_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB145_6
|
|
; GCN1-NEXT: .LBB145_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB145_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[38:39], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB145_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
|
|
; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v2
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[38:39]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB145_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[38:39]
|
|
; GCN1-NEXT: s_branch .LBB145_2
|
|
; GCN1-NEXT: .LBB145_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v0
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB145_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB145_6
|
|
; GCN2-NEXT: .LBB145_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB145_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[38:39], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB145_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
|
|
; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v2
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[38:39]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB145_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[38:39]
|
|
; GCN2-NEXT: s_branch .LBB145_2
|
|
; GCN2-NEXT: .LBB145_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v0
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB145_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB145_6
|
|
; GCN3-NEXT: .LBB145_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB145_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[38:39], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB145_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
|
|
; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v2
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[38:39]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB145_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[38:39]
|
|
; GCN3-NEXT: s_branch .LBB145_2
|
|
; GCN3-NEXT: .LBB145_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v0
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v1, s[36:37]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s38, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s39, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s39, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB146_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB146_6
|
|
; GCN1-NEXT: .LBB146_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB146_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s38, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s39, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s38
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s39
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[40:41], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB146_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
|
|
; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v2
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[40:41], vcc, s[40:41]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[40:41]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB146_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[40:41]
|
|
; GCN1-NEXT: s_branch .LBB146_2
|
|
; GCN1-NEXT: .LBB146_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[38:39], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s38, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v0
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s38, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s39, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s39, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB146_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB146_6
|
|
; GCN2-NEXT: .LBB146_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB146_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s38, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s39, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s38
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s39
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[40:41], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB146_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
|
|
; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v2
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[40:41], vcc, s[40:41]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[40:41]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB146_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[40:41]
|
|
; GCN2-NEXT: s_branch .LBB146_2
|
|
; GCN2-NEXT: .LBB146_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[38:39], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s38, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v0
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s38, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s39, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s39, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB146_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB146_6
|
|
; GCN3-NEXT: .LBB146_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB146_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s38
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s39
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[40:41], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB146_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
|
|
; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v2
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[40:41], vcc, s[40:41]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[40:41]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB146_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[40:41]
|
|
; GCN3-NEXT: s_branch .LBB146_2
|
|
; GCN3-NEXT: .LBB146_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[38:39], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s38, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v0
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v1, s[36:37]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB147_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[38:39], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB147_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v8
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[38:39]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB147_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[38:39]
|
|
; GCN1-NEXT: s_branch .LBB147_6
|
|
; GCN1-NEXT: .LBB147_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB147_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e64 v6, s[36:37], -1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB147_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB147_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[38:39], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB147_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v8
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[38:39]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB147_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[38:39]
|
|
; GCN2-NEXT: s_branch .LBB147_6
|
|
; GCN2-NEXT: .LBB147_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB147_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e64 v6, s[36:37], -1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB147_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB147_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[38:39], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB147_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v8
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[38:39]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB147_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[38:39]
|
|
; GCN3-NEXT: s_branch .LBB147_6
|
|
; GCN3-NEXT: .LBB147_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB147_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e64 v5, s[36:37], -1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v6, s[36:37], -1, v1, s[36:37]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB147_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s38, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s39, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s39, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB148_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s38, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s39, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s38
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s39
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[40:41], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB148_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v8
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[40:41], vcc, s[40:41]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[40:41]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB148_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[40:41]
|
|
; GCN1-NEXT: s_branch .LBB148_6
|
|
; GCN1-NEXT: .LBB148_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB148_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[38:39], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s38, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e64 v6, s[36:37], -1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN1-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB148_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s38, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s39, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s39, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB148_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s38, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s39, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s38
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s39
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[40:41], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB148_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v8
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[40:41], vcc, s[40:41]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[40:41]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB148_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[40:41]
|
|
; GCN2-NEXT: s_branch .LBB148_6
|
|
; GCN2-NEXT: .LBB148_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB148_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[38:39], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s38, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e64 v6, s[36:37], -1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN2-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB148_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s38, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s39, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s39, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB148_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s38
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s39
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[40:41], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: .LBB148_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
|
|
; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v8
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[40:41], vcc, s[40:41]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[40:41]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB148_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[40:41]
|
|
; GCN3-NEXT: s_branch .LBB148_6
|
|
; GCN3-NEXT: .LBB148_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB148_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[38:39], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s38, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e64 v5, s[36:37], -1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v6, s[36:37], -1, v1, s[36:37]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB148_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB149_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB149_4
|
|
; GCN1-NEXT: .LBB149_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB149_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB149_2
|
|
; GCN1-NEXT: .LBB149_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB149_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB149_4
|
|
; GCN2-NEXT: .LBB149_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB149_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB149_2
|
|
; GCN2-NEXT: .LBB149_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB149_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB149_4
|
|
; GCN3-NEXT: .LBB149_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB149_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB149_2
|
|
; GCN3-NEXT: .LBB149_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB150_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB150_4
|
|
; GCN1-NEXT: .LBB150_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB150_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB150_2
|
|
; GCN1-NEXT: .LBB150_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7]
|
|
; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB150_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB150_4
|
|
; GCN2-NEXT: .LBB150_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB150_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB150_2
|
|
; GCN2-NEXT: .LBB150_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7]
|
|
; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB150_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB150_4
|
|
; GCN3-NEXT: .LBB150_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB150_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB150_2
|
|
; GCN3-NEXT: .LBB150_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7]
|
|
; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[8:9]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
!0 = !{}
|