; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG-TRUE16
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL-TRUE16
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX12,GFX12-SDAG-TRUE16
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-TRUE16
; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16

define amdgpu_kernel void @raw_atomic_buffer_load_i32(<4 x i32> %addr) {
; GFX11-LABEL: raw_atomic_buffer_load_i32:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_mov_b32 s4, 0
; GFX11-NEXT:  .LBB0_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT:    s_cbranch_execnz .LBB0_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_atomic_buffer_load_i32:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:  .LBB0_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_execnz .LBB0_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 0, i32 0, i32 1)
  %cmp = icmp eq i32 %load, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}

define amdgpu_kernel void @raw_atomic_buffer_load_i32_off(<4 x i32> %addr) {
; GFX11-LABEL: raw_atomic_buffer_load_i32_off:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_mov_b32 s4, 0
; GFX11-NEXT:  .LBB1_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT:    s_cbranch_execnz .LBB1_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_atomic_buffer_load_i32_off:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:  .LBB1_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_execnz .LBB1_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 0, i32 0, i32 1)
  %cmp = icmp eq i32 %load, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}
define amdgpu_kernel void @raw_atomic_buffer_load_i32_soff(<4 x i32> %addr) {
; GFX11-LABEL: raw_atomic_buffer_load_i32_soff:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_mov_b32 s4, 0
; GFX11-NEXT:  .LBB2_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT:    s_cbranch_execnz .LBB2_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_atomic_buffer_load_i32_soff:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    s_mov_b32 s5, 4
; GFX12-NEXT:  .LBB2_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 4, i32 4, i32 1)
  %cmp = icmp eq i32 %load, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}
define amdgpu_kernel void @raw_atomic_buffer_load_i32_dlc(<4 x i32> %addr) {
; GFX11-LABEL: raw_atomic_buffer_load_i32_dlc:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_mov_b32 s4, 0
; GFX11-NEXT:  .LBB3_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT:    s_cbranch_execnz .LBB3_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_atomic_buffer_load_i32_dlc:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:  .LBB3_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32> %addr, i32 4, i32 0, i32 4)
  %cmp = icmp eq i32 %load, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}

define amdgpu_kernel void @raw_nonatomic_buffer_load_i32(<4 x i32> %addr) {
; GFX11-LABEL: raw_nonatomic_buffer_load_i32:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
; GFX11-NEXT:    s_mov_b32 s0, 0
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-NEXT:  .LBB4_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT:    s_or_b32 s0, s1, s0
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT:    s_cbranch_execnz .LBB4_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_nonatomic_buffer_load_i32:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s0, 0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-NEXT:  .LBB4_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_or_b32 s0, s1, s0
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %addr, i32 4, i32 0, i32 1)
  %cmp = icmp eq i32 %load, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}

define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
; GFX11-LABEL: raw_atomic_buffer_load_i64:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-NEXT:    s_mov_b32 s4, 0
; GFX11-NEXT:  .LBB5_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT:    s_cbranch_execnz .LBB5_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_atomic_buffer_load_i64:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    v_mov_b32_e32 v1, 0
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:  .LBB5_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %id.zext = zext i32 %id to i64
  br label %bb1
bb1:
  %load = call i64 @llvm.amdgcn.raw.atomic.buffer.load.i64(<4 x i32> %addr, i32 4, i32 0, i32 1)
  %cmp = icmp eq i64 %load, %id.zext
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}

define amdgpu_kernel void @raw_atomic_buffer_load_v2i16(<4 x i32> %addr) {
; GFX11-LABEL: raw_atomic_buffer_load_v2i16:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_mov_b32 s4, 0
; GFX11-NEXT:  .LBB6_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT:    s_cbranch_execnz .LBB6_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_atomic_buffer_load_v2i16:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:  .LBB6_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call <2 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v2i16(<4 x i32> %addr, i32 0, i32 0, i32 1)
  %bitcast = bitcast <2 x i16> %load to i32
  %cmp = icmp eq i32 %bitcast, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}

define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX11-SDAG-TRUE16:       ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
; GFX11-SDAG-TRUE16-NEXT:  .LBB7_1: ; %bb1
; GFX11-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
; GFX11-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
; GFX11-SDAG-TRUE16-NEXT:    s_endpgm
;
; GFX11-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX11-FAKE16:       ; %bb.0: ; %bb
; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT:    s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT:  .LBB7_1: ; %bb1
; GFX11-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX11-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT:    s_cbranch_execnz .LBB7_1
; GFX11-FAKE16-NEXT:  ; %bb.2: ; %bb2
; GFX11-FAKE16-NEXT:    s_endpgm
;
; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX11-GISEL-TRUE16:       ; %bb.0: ; %bb
; GFX11-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
; GFX11-GISEL-TRUE16-NEXT:  .LBB7_1: ; %bb1
; GFX11-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v2.l
; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
; GFX11-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
; GFX11-GISEL-TRUE16-NEXT:    s_endpgm
;
; GFX11-GISEL-LABEL: raw_atomic_buffer_load_v4i16:
; GFX11-GISEL:       ; %bb.0: ; %bb
; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-GISEL-NEXT:    s_mov_b32 s4, 0
; GFX11-GISEL-NEXT:  .LBB7_1: ; %bb1
; GFX11-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s5, v1
; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s6, v2
; GFX11-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
; GFX11-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
; GFX11-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-GISEL-NEXT:    s_cbranch_execnz .LBB7_1
; GFX11-GISEL-NEXT:  ; %bb.2: ; %bb2
; GFX11-GISEL-NEXT:    s_endpgm
;
; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX12-SDAG-TRUE16:       ; %bb.0: ; %bb
; GFX12-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT:    s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
; GFX12-SDAG-TRUE16-NEXT:  .LBB7_1: ; %bb1
; GFX12-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
; GFX12-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
; GFX12-SDAG-TRUE16-NEXT:    s_endpgm
;
; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX12-FAKE16:       ; %bb.0: ; %bb
; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-FAKE16-NEXT:    s_wait_xcnt 0x0
; GFX12-FAKE16-NEXT:    s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT:  .LBB7_1: ; %bb1
; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
; GFX12-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-FAKE16-NEXT:    s_cbranch_execnz .LBB7_1
; GFX12-FAKE16-NEXT:  ; %bb.2: ; %bb2
; GFX12-FAKE16-NEXT:    s_endpgm
;
; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX12-GISEL-TRUE16:       ; %bb.0: ; %bb
; GFX12-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT:    s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
; GFX12-GISEL-TRUE16-NEXT:  .LBB7_1: ; %bb1
; GFX12-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
; GFX12-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
; GFX12-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
; GFX12-GISEL-TRUE16-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call <4 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v4i16(<4 x i32> %addr, i32 4, i32 0, i32 1)
  %shortened = shufflevector <4 x i16> %load, <4 x i16> poison, <2 x i32> <i32 0, i32 2>
  %bitcast = bitcast <2 x i16> %shortened to i32
  %cmp = icmp eq i32 %bitcast, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}

define amdgpu_kernel void @raw_atomic_buffer_load_v4i32(<4 x i32> %addr) {
; GFX11-LABEL: raw_atomic_buffer_load_v4i32:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_mov_b32 s4, 0
; GFX11-NEXT:  .LBB8_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v0
; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT:    s_cbranch_execnz .LBB8_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_atomic_buffer_load_v4i32:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:  .LBB8_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call <4 x i32> @llvm.amdgcn.raw.atomic.buffer.load.v4i32(<4 x i32> %addr, i32 4, i32 0, i32 1)
  %extracted = extractelement <4 x i32> %load, i32 3
  %cmp = icmp eq i32 %extracted, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}

define amdgpu_kernel void @raw_atomic_buffer_load_ptr(<4 x i32> %addr) {
; GFX11-LABEL: raw_atomic_buffer_load_ptr:
; GFX11:       ; %bb.0: ; %bb
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_mov_b32 s4, 0
; GFX11-NEXT:  .LBB9_1: ; %bb1
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    flat_load_b32 v1, v[1:2]
; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT:    s_cbranch_execnz .LBB9_1
; GFX11-NEXT:  ; %bb.2: ; %bb2
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: raw_atomic_buffer_load_ptr:
; GFX12:       ; %bb.0: ; %bb
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_wait_xcnt 0x0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:  .LBB9_1: ; %bb1
; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_load_b32 v1, v[2:3]
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
; GFX12-NEXT:  ; %bb.2: ; %bb2
; GFX12-NEXT:    s_endpgm
bb:
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  br label %bb1
bb1:
  %load = call ptr @llvm.amdgcn.raw.atomic.buffer.load.ptr(<4 x i32> %addr, i32 4, i32 0, i32 1)
  %elem = load i32, ptr %load
  %cmp = icmp eq i32 %elem, %id
  br i1 %cmp, label %bb1, label %bb2
bb2:
  ret void
}

; Function Attrs: nounwind readonly
declare i32 @llvm.amdgcn.raw.atomic.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg)
declare i64 @llvm.amdgcn.raw.atomic.buffer.load.i64(<4 x i32>, i32, i32, i32 immarg)
declare <2 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v2i16(<4 x i32>, i32, i32, i32 immarg)
declare <4 x i16> @llvm.amdgcn.raw.atomic.buffer.load.v4i16(<4 x i32>, i32, i32, i32 immarg)
declare <4 x i32> @llvm.amdgcn.raw.atomic.buffer.load.v4i32(<4 x i32>, i32, i32, i32 immarg)
declare ptr @llvm.amdgcn.raw.atomic.buffer.load.ptr(<4 x i32>, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg)
declare i32 @llvm.amdgcn.workitem.id.x()