shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
Jeffrey Byrnes 7364203924
Reapply "[AMDGPU] Add HWUI pressure heuristics to coexec strategy (#184929)" (#189121) 
Reland https://github.com/llvm/llvm-project/pull/184929 after fixing
some issues in the NDEBUG builds.

3a640ee is unchanged from the previously approved PR, the unreviewed
portion of this PR is 9cabd8d
2026-03-30 12:18:29 -07:00

607 lines
41 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -amdgpu-sched-strategy=coexec --enable-post-misched=0 --verify-misched < %s | FileCheck -check-prefix=COEXEC %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s
define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out, i1 %br0, i32 %delta) local_unnamed_addr #0 {
; COEXEC-LABEL: ds_wmma:
; COEXEC: ; %bb.0: ; %entry
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
; COEXEC-NEXT: s_clause 0x1
; COEXEC-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; COEXEC-NEXT: s_delay_alu instid0(VALU_DEP_1)
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
; COEXEC-NEXT: v_mov_b32_e32 v5, v0
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
; COEXEC-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; COEXEC-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
; COEXEC-NEXT: v_mov_b32_e32 v6, v0
; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v7
; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
; COEXEC-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v24, v0
; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v17, v0
; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v10, v0
; COEXEC-NEXT: v_dual_mov_b32 v18, v0 :: v_dual_mov_b32 v26, v0
; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v19, v0
; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v12, v0
; COEXEC-NEXT: v_dual_mov_b32 v20, v0 :: v_dual_mov_b32 v28, v0
; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v21, v0
; COEXEC-NEXT: v_dual_mov_b32 v29, v0 :: v_dual_mov_b32 v14, v0
; COEXEC-NEXT: v_dual_mov_b32 v22, v0 :: v_dual_mov_b32 v30, v0
; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v23, v0
; COEXEC-NEXT: v_mov_b32_e32 v31, v0
; COEXEC-NEXT: .LBB0_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v92, s2
; COEXEC-NEXT: s_add_co_i32 s2, s2, s1
; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v92 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v92
; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v92 offset:192
; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v92 offset:64
; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v92 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v92 offset:256
; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v92 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v92 offset:320
; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v92 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v92 offset:512
; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v92 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v92 offset:576
; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v92 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v92 offset:768
; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v92 offset:960
; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v92 offset:832
; COEXEC-NEXT: s_wait_dscnt 0xc
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31]
; COEXEC-NEXT: s_wait_dscnt 0x8
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[48:55], v[16:23]
; COEXEC-NEXT: s_wait_dscnt 0x4
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[80:87], v[0:7]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[40:47], v[32:39], v[24:31]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[56:63], v[48:55], v[16:23]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[72:79], v[64:71], v[8:15]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[88:95], v[80:87], v[0:7]
; COEXEC-NEXT: s_cbranch_vccnz .LBB0_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v32, 0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[0:1]
; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; COEXEC-NEXT: s_endpgm
;
; GCN-LABEL: ds_wmma:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; GCN-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_bitcmp1_b32 s0, 0
; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
; GCN-NEXT: s_xor_b32 s0, s0, -1
; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
; GCN-NEXT: v_cndmask_b32_e64 v24, 0, 1, s0
; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v25, v0
; GCN-NEXT: v_mov_b32_e32 v26, v0
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v24
; GCN-NEXT: v_dual_mov_b32 v24, v0 :: v_dual_mov_b32 v27, v0
; GCN-NEXT: v_dual_mov_b32 v28, v0 :: v_dual_mov_b32 v29, v0
; GCN-NEXT: v_dual_mov_b32 v30, v0 :: v_dual_mov_b32 v31, v0
; GCN-NEXT: .LBB0_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
; GCN-NEXT: v_mov_b32_e32 v92, s2
; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GCN-NEXT: s_add_co_i32 s2, s2, s1
; GCN-NEXT: ds_load_tr16_b128 v[32:35], v92
; GCN-NEXT: ds_load_tr16_b128 v[36:39], v92 offset:64
; GCN-NEXT: ds_load_tr16_b128 v[40:43], v92 offset:128
; GCN-NEXT: ds_load_tr16_b128 v[44:47], v92 offset:192
; GCN-NEXT: ds_load_tr16_b128 v[48:51], v92 offset:256
; GCN-NEXT: ds_load_tr16_b128 v[52:55], v92 offset:320
; GCN-NEXT: ds_load_tr16_b128 v[56:59], v92 offset:384
; GCN-NEXT: ds_load_tr16_b128 v[60:63], v92 offset:448
; GCN-NEXT: ds_load_tr16_b128 v[64:67], v92 offset:512
; GCN-NEXT: ds_load_tr16_b128 v[68:71], v92 offset:576
; GCN-NEXT: ds_load_tr16_b128 v[72:75], v92 offset:640
; GCN-NEXT: ds_load_tr16_b128 v[76:79], v92 offset:704
; GCN-NEXT: ds_load_tr16_b128 v[80:83], v92 offset:768
; GCN-NEXT: ds_load_tr16_b128 v[84:87], v92 offset:832
; GCN-NEXT: ds_load_tr16_b128 v[88:91], v92 offset:896
; GCN-NEXT: ds_load_tr16_b128 v[92:95], v92 offset:960
; GCN-NEXT: s_wait_dscnt 0xc
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; GCN-NEXT: s_wait_dscnt 0x8
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; GCN-NEXT: s_wait_dscnt 0x4
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; GCN-NEXT: s_wait_dscnt 0x0
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; GCN-NEXT: s_cbranch_vccnz .LBB0_1
; GCN-NEXT: ; %bb.2: ; %end
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GCN-NEXT: v_nop
; GCN-NEXT: v_mov_b32_e32 v32, 0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x7
; GCN-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
; GCN-NEXT: global_store_b128 v32, v[24:27], s[0:1]
; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
; GCN-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
; GCN-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GCN-NEXT: s_endpgm
entry:
br label %loop
loop:
%baseOff = phi i32 [ 0, %entry ], [ %newBaseOff, %loop ]
%wvec0 = phi <8 x float> [ <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %wmma01, %loop ]
%wvec1 = phi <8 x float> [ <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %wmma11, %loop ]
%wvec2 = phi <8 x float> [ <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %wmma21, %loop ]
%wvec3 = phi <8 x float> [ <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %wmma31, %loop ]
%p0 = getelementptr inbounds nuw i8, ptr addrspace(3) %base, i32 %baseOff
%p1 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 64
%p2 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 128
%p3 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 192
%p4 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 256
%p5 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 320
%p6 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 384
%p7 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 448
%p8 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 512
%p9 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 576
%p10 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 640
%p11 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 704
%p12 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 768
%p13 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 832
%p14 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 896
%p15 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 960
%l0 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %p0)
%l1 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p1)
%l2 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p2)
%l3 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p3)
%l4 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p4)
%l5 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p5)
%l6 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p6)
%l7 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p7)
%l8 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p8)
%l9 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p9)
%l10 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p10)
%l11 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p11)
%l12 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p12)
%l13 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p13)
%l14 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p14)
%l15 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p15)
%vec0 = shufflevector <8 x half> %l0, <8 x half> %l1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec1 = shufflevector <8 x half> %l2, <8 x half> %l3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec2 = shufflevector <8 x half> %l4, <8 x half> %l5, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec3 = shufflevector <8 x half> %l6, <8 x half> %l7, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec4 = shufflevector <8 x half> %l8, <8 x half> %l9, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec5 = shufflevector <8 x half> %l10, <8 x half> %l11, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec6 = shufflevector <8 x half> %l12, <8 x half> %l13, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec7 = shufflevector <8 x half> %l14, <8 x half> %l15, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%wmma00 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec0, i1 false, <16 x half> %vec1, i16 0, <8 x float> %wvec0, i1 false, i1 false)
%wmma01 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec0, i1 false, <16 x half> %vec1, i16 0, <8 x float> %wmma00, i1 false, i1 false)
%wmma10 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec2, i1 false, <16 x half> %vec3, i16 0, <8 x float> %wvec1, i1 false, i1 false)
%wmma11 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec2, i1 false, <16 x half> %vec3, i16 0, <8 x float> %wmma10, i1 false, i1 false)
%wmma20 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec4, i1 false, <16 x half> %vec5, i16 0, <8 x float> %wvec2, i1 false, i1 false)
%wmma21 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec4, i1 false, <16 x half> %vec5, i16 0, <8 x float> %wmma20, i1 false, i1 false)
%wmma30 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec6, i1 false, <16 x half> %vec7, i16 0, <8 x float> %wvec3, i1 false, i1 false)
%wmma31 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec6, i1 false, <16 x half> %vec7, i16 0, <8 x float> %wmma30, i1 false, i1 false)
%newBaseOff = or disjoint i32 %baseOff, %delta
br i1 %br0, label %loop, label %end
end:
%out1 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 128
%out2 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 256
%out3 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 384
store <8 x float> %wmma01, ptr addrspace(1) %out, align 16
store <8 x float> %wmma11, ptr addrspace(1) %out1, align 16
store <8 x float> %wmma21, ptr addrspace(1) %out2, align 16
store <8 x float> %wmma31, ptr addrspace(1) %out3, align 16
ret void
}
define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace(3) %base1, ptr addrspace(1) %out, i1 %br0, i32 %delta) local_unnamed_addr #0 {
; COEXEC-LABEL: ds_wmma_permute:
; COEXEC: ; %bb.0: ; %entry
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; COEXEC-NEXT: s_mov_b32 s6, 0
; COEXEC-NEXT: s_clause 0x1
; COEXEC-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
; COEXEC-NEXT: s_delay_alu instid0(VALU_DEP_1)
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_mov_b32_e32 v3, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
; COEXEC-NEXT: v_mov_b32_e32 v4, v0
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
; COEXEC-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
; COEXEC-NEXT: v_mov_b32_e32 v5, v0
; COEXEC-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
; COEXEC-NEXT: v_mov_b32_e32 v6, v0
; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v7
; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
; COEXEC-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v24, v0
; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v17, v0
; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v10, v0
; COEXEC-NEXT: v_dual_mov_b32 v18, v0 :: v_dual_mov_b32 v26, v0
; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v19, v0
; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v12, v0
; COEXEC-NEXT: v_dual_mov_b32 v20, v0 :: v_dual_mov_b32 v28, v0
; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v21, v0
; COEXEC-NEXT: v_dual_mov_b32 v29, v0 :: v_dual_mov_b32 v14, v0
; COEXEC-NEXT: v_dual_mov_b32 v22, v0 :: v_dual_mov_b32 v30, v0
; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v23, v0
; COEXEC-NEXT: v_mov_b32_e32 v31, v0
; COEXEC-NEXT: .LBB1_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
; COEXEC-NEXT: s_add_co_i32 s7, s2, s6
; COEXEC-NEXT: s_add_co_i32 s8, s3, s6
; COEXEC-NEXT: s_add_co_i32 s6, s6, s1
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v124, s7
; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; COEXEC-NEXT: v_mov_b32_e32 v156, s8
; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v124
; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v124 offset:64
; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v156
; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v156 offset:64
; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v124 offset:256
; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v156 offset:256
; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v124 offset:320
; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v156 offset:320
; COEXEC-NEXT: ds_load_tr16_b128 v[64:67], v124 offset:512
; COEXEC-NEXT: ds_load_tr16_b128 v[72:75], v156 offset:512
; COEXEC-NEXT: ds_load_tr16_b128 v[68:71], v124 offset:576
; COEXEC-NEXT: ds_load_tr16_b128 v[76:79], v156 offset:576
; COEXEC-NEXT: ds_load_tr16_b128 v[80:83], v124 offset:768
; COEXEC-NEXT: ds_load_tr16_b128 v[88:91], v156 offset:768
; COEXEC-NEXT: ds_load_tr16_b128 v[84:87], v124 offset:832
; COEXEC-NEXT: ds_load_tr16_b128 v[92:95], v156 offset:832
; COEXEC-NEXT: ds_load_tr16_b128 v[96:99], v124 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[104:107], v124 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v124 offset:640
; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128
; COEXEC-NEXT: ds_load_tr16_b128 v[136:139], v156 offset:384
; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640
; COEXEC-NEXT: s_wait_dscnt 0x13
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896
; COEXEC-NEXT: ds_load_tr16_b128 v[100:103], v124 offset:192
; COEXEC-NEXT: ds_load_tr16_b128 v[108:111], v124 offset:448
; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v124 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960
; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192
; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448
; COEXEC-NEXT: s_wait_dscnt 0x16
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704
; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960
; COEXEC-NEXT: s_wait_dscnt 0x14
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x10
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; COEXEC-NEXT: s_wait_dscnt 0x3
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31]
; COEXEC-NEXT: s_wait_dscnt 0x2
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23]
; COEXEC-NEXT: s_wait_dscnt 0x1
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15]
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[96:103], v[128:135], v[24:31]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[104:111], v[136:143], v[16:23]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[112:119], v[144:151], v[8:15]
; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[120:127], v[152:159], v[0:7]
; COEXEC-NEXT: s_cbranch_vccnz .LBB1_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: v_mov_b32_e32 v32, 0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[0:1]
; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; COEXEC-NEXT: s_endpgm
;
; GCN-LABEL: ds_wmma_permute:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_bitcmp1_b32 s0, 0
; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
; GCN-NEXT: s_xor_b32 s0, s0, -1
; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
; GCN-NEXT: v_cndmask_b32_e64 v24, 0, 1, s0
; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v25, v0
; GCN-NEXT: v_mov_b32_e32 v26, v0
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GCN-NEXT: v_cmp_ne_u32_e64 s0, 1, v24
; GCN-NEXT: v_dual_mov_b32 v24, v0 :: v_dual_mov_b32 v27, v0
; GCN-NEXT: v_dual_mov_b32 v28, v0 :: v_dual_mov_b32 v29, v0
; GCN-NEXT: v_dual_mov_b32 v30, v0 :: v_dual_mov_b32 v31, v0
; GCN-NEXT: .LBB1_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: s_add_co_i32 s7, s2, s6
; GCN-NEXT: s_add_co_i32 s8, s3, s6
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT: v_dual_mov_b32 v96, s7 :: v_dual_mov_b32 v97, s8
; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
; GCN-NEXT: s_add_co_i32 s6, s6, s1
; GCN-NEXT: ds_load_tr16_b128 v[32:35], v96
; GCN-NEXT: ds_load_tr16_b128 v[36:39], v96 offset:64
; GCN-NEXT: ds_load_tr16_b128 v[40:43], v97
; GCN-NEXT: ds_load_tr16_b128 v[44:47], v97 offset:64
; GCN-NEXT: ds_load_tr16_b128 v[48:51], v96 offset:256
; GCN-NEXT: ds_load_tr16_b128 v[52:55], v96 offset:320
; GCN-NEXT: ds_load_tr16_b128 v[56:59], v97 offset:256
; GCN-NEXT: ds_load_tr16_b128 v[60:63], v97 offset:320
; GCN-NEXT: ds_load_tr16_b128 v[64:67], v96 offset:512
; GCN-NEXT: ds_load_tr16_b128 v[68:71], v96 offset:576
; GCN-NEXT: ds_load_tr16_b128 v[72:75], v97 offset:512
; GCN-NEXT: ds_load_tr16_b128 v[76:79], v97 offset:576
; GCN-NEXT: ds_load_tr16_b128 v[80:83], v96 offset:768
; GCN-NEXT: ds_load_tr16_b128 v[84:87], v96 offset:832
; GCN-NEXT: ds_load_tr16_b128 v[88:91], v97 offset:768
; GCN-NEXT: ds_load_tr16_b128 v[92:95], v97 offset:832
; GCN-NEXT: s_wait_dscnt 0xc
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; GCN-NEXT: s_wait_dscnt 0x8
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; GCN-NEXT: s_wait_dscnt 0x4
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; GCN-NEXT: s_wait_dscnt 0x0
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; GCN-NEXT: ds_load_tr16_b128 v[32:35], v96 offset:128
; GCN-NEXT: ds_load_tr16_b128 v[36:39], v96 offset:192
; GCN-NEXT: ds_load_tr16_b128 v[40:43], v97 offset:128
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; GCN-NEXT: ds_load_tr16_b128 v[44:47], v97 offset:192
; GCN-NEXT: ds_load_tr16_b128 v[48:51], v96 offset:384
; GCN-NEXT: ds_load_tr16_b128 v[52:55], v96 offset:448
; GCN-NEXT: ds_load_tr16_b128 v[56:59], v97 offset:384
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; GCN-NEXT: ds_load_tr16_b128 v[60:63], v97 offset:448
; GCN-NEXT: ds_load_tr16_b128 v[64:67], v96 offset:640
; GCN-NEXT: ds_load_tr16_b128 v[68:71], v96 offset:704
; GCN-NEXT: ds_load_tr16_b128 v[72:75], v97 offset:640
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; GCN-NEXT: ds_load_tr16_b128 v[76:79], v97 offset:704
; GCN-NEXT: ds_load_tr16_b128 v[80:83], v96 offset:896
; GCN-NEXT: ds_load_tr16_b128 v[84:87], v96 offset:960
; GCN-NEXT: ds_load_tr16_b128 v[88:91], v97 offset:896
; GCN-NEXT: ds_load_tr16_b128 v[92:95], v97 offset:960
; GCN-NEXT: s_wait_dscnt 0xc
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; GCN-NEXT: s_wait_dscnt 0x8
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; GCN-NEXT: s_wait_dscnt 0x4
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; GCN-NEXT: s_wait_dscnt 0x0
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15]
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; GCN-NEXT: s_cbranch_vccnz .LBB1_1
; GCN-NEXT: ; %bb.2: ; %end
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GCN-NEXT: v_nop
; GCN-NEXT: v_mov_b32_e32 v32, 0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x7
; GCN-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
; GCN-NEXT: global_store_b128 v32, v[24:27], s[0:1]
; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
; GCN-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
; GCN-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GCN-NEXT: s_endpgm
entry:
br label %loop
loop:
%baseOff = phi i32 [ 0, %entry ], [ %newBaseOff, %loop ]
%wvec0 = phi <8 x float> [ <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %bwmma01, %loop ]
%wvec1 = phi <8 x float> [ <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %bwmma11, %loop ]
%wvec2 = phi <8 x float> [ <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %bwmma21, %loop ]
%wvec3 = phi <8 x float> [ <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, %entry ], [ %bwmma31, %loop ]
%p0 = getelementptr inbounds nuw i8, ptr addrspace(3) %base, i32 %baseOff
%p1 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 64
%p2 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 128
%p3 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 192
%p4 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 256
%p5 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 320
%p6 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 384
%p7 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 448
%p8 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 512
%p9 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 576
%p10 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 640
%p11 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 704
%p12 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 768
%p13 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 832
%p14 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 896
%p15 = getelementptr inbounds nuw i8, ptr addrspace(3) %p0, i32 960
%bp0 = getelementptr inbounds nuw i8, ptr addrspace(3) %base1, i32 %baseOff
%bp1 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 64
%bp2 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 128
%bp3 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 192
%bp4 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 256
%bp5 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 320
%bp6 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 384
%bp7 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 448
%bp8 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 512
%bp9 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 576
%bp10 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 640
%bp11 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 704
%bp12 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 768
%bp13 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 832
%bp14 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 896
%bp15 = getelementptr inbounds nuw i8, ptr addrspace(3) %bp0, i32 960
%l0 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %p0)
%l1 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p1)
%l2 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p2)
%l3 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p3)
%l4 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p4)
%l5 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p5)
%l6 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p6)
%l7 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p7)
%l8 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p8)
%l9 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p9)
%l10 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p10)
%l11 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p11)
%l12 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p12)
%l13 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p13)
%l14 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p14)
%l15 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %p15)
%bl0 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %bp0)
%bl1 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp1)
%bl2 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp2)
%bl3 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp3)
%bl4 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp4)
%bl5 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp5)
%bl6 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp6)
%bl7 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp7)
%bl8 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp8)
%bl9 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp9)
%bl10 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp10)
%bl11 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp11)
%bl12 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp12)
%bl13 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp13)
%bl14 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp14)
%bl15 = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) nonnull %bp15)
%vec0 = shufflevector <8 x half> %l0, <8 x half> %l1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec1 = shufflevector <8 x half> %l2, <8 x half> %l3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec2 = shufflevector <8 x half> %l4, <8 x half> %l5, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec3 = shufflevector <8 x half> %l6, <8 x half> %l7, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec4 = shufflevector <8 x half> %l8, <8 x half> %l9, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec5 = shufflevector <8 x half> %l10, <8 x half> %l11, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec6 = shufflevector <8 x half> %l12, <8 x half> %l13, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%vec7 = shufflevector <8 x half> %l14, <8 x half> %l15, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%bvec0 = shufflevector <8 x half> %bl0, <8 x half> %bl1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%bvec1 = shufflevector <8 x half> %bl2, <8 x half> %bl3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%bvec2 = shufflevector <8 x half> %bl4, <8 x half> %bl5, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%bvec3 = shufflevector <8 x half> %bl6, <8 x half> %bl7, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%bvec4 = shufflevector <8 x half> %bl8, <8 x half> %bl9, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%bvec5 = shufflevector <8 x half> %bl10, <8 x half> %bl11, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%bvec6 = shufflevector <8 x half> %bl12, <8 x half> %bl13, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%bvec7 = shufflevector <8 x half> %bl14, <8 x half> %bl15, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%wmma00 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec0, i1 false, <16 x half> %bvec0, i16 0, <8 x float> %wvec0, i1 false, i1 false)
%bwmma00 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec0, i1 false, <16 x half> %bvec0, i16 0, <8 x float> %wmma00, i1 false, i1 false)
%wmma01 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec1, i1 false, <16 x half> %bvec1, i16 0, <8 x float> %bwmma00, i1 false, i1 false)
%bwmma01 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec1, i1 false, <16 x half> %bvec1, i16 0, <8 x float> %wmma01, i1 false, i1 false)
%wmma10 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec2, i1 false, <16 x half> %bvec2, i16 0, <8 x float> %wvec1, i1 false, i1 false)
%bwmma10 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec2, i1 false, <16 x half> %bvec2, i16 0, <8 x float> %wmma10, i1 false, i1 false)
%wmma11 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec3, i1 false, <16 x half> %bvec3, i16 0, <8 x float> %bwmma10, i1 false, i1 false)
%bwmma11 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec3, i1 false, <16 x half> %bvec3, i16 0, <8 x float> %wmma11, i1 false, i1 false)
%wmma20 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec4, i1 false, <16 x half> %bvec4, i16 0, <8 x float> %wvec2, i1 false, i1 false)
%bwmma20 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec4, i1 false, <16 x half> %bvec4, i16 0, <8 x float> %wmma20, i1 false, i1 false)
%wmma21 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec5, i1 false, <16 x half> %bvec5, i16 0, <8 x float> %bwmma20, i1 false, i1 false)
%bwmma21 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec5, i1 false, <16 x half> %bvec5, i16 0, <8 x float> %wmma21, i1 false, i1 false)
%wmma30 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec6, i1 false, <16 x half> %bvec6, i16 0, <8 x float> %wvec3, i1 false, i1 false)
%bwmma30 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec6, i1 false, <16 x half> %bvec6, i16 0, <8 x float> %wmma30, i1 false, i1 false)
%wmma31 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec7, i1 false, <16 x half> %bvec7, i16 0, <8 x float> %bwmma30, i1 false, i1 false)
%bwmma31 = tail call <8 x float> @llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1 false, <16 x half> %vec7, i1 false, <16 x half> %bvec7, i16 0, <8 x float> %wmma31, i1 false, i1 false)
%newBaseOff = or disjoint i32 %baseOff, %delta
br i1 %br0, label %loop, label %end
end:
%out1 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 128
%out2 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 256
%out3 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i32 384
store <8 x float> %bwmma01, ptr addrspace(1) %out, align 16
store <8 x float> %bwmma11, ptr addrspace(1) %out1, align 16
store <8 x float> %bwmma21, ptr addrspace(1) %out2, align 16
store <8 x float> %bwmma31, ptr addrspace(1) %out3, align 16
ret void
}
attributes #0 = { "amdgpu-flat-work-group-size"="32,32" "amdgpu-waves-per-eu"="1,1" }
Reference in New Issue View Git Blame Copy Permalink