056e5a32c8
Keep bf16/f16 values encoded as the low half of a 32-bit register, instead of promoting to float. This avoids unwanted FP effects from the fpext/fptrunc which should not be implied by just passing an argument. This also fixes ABI divergence between SelectionDAG and GlobalISel. I've wanted to make this change for ages, and failed the last few times. The main complication was the hack to return shader integer types in SGPRs, which now needs to inspect the underlying IR type.
154 lines
5.6 KiB
LLVM
154 lines
5.6 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
|
|
; RUN: llc -mtriple=amdgcn -mcpu=hawaii < %s | FileCheck --check-prefix=GFX7 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=GFX8 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefix=GFX11 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefix=GFX12 %s
|
|
|
|
define amdgpu_ps void @buffer_store_bf16(ptr addrspace(8) inreg %rsrc, bfloat %data, i32 %offset) {
|
|
; GFX7-LABEL: buffer_store_bf16:
|
|
; GFX7: ; %bb.0:
|
|
; GFX7-NEXT: buffer_store_short v0, v1, s[0:3], 0 offen
|
|
; GFX7-NEXT: s_endpgm
|
|
;
|
|
; GFX8-LABEL: buffer_store_bf16:
|
|
; GFX8: ; %bb.0:
|
|
; GFX8-NEXT: buffer_store_short v0, v1, s[0:3], 0 offen
|
|
; GFX8-NEXT: s_endpgm
|
|
;
|
|
; GFX9-LABEL: buffer_store_bf16:
|
|
; GFX9: ; %bb.0:
|
|
; GFX9-NEXT: buffer_store_short v0, v1, s[0:3], 0 offen
|
|
; GFX9-NEXT: s_endpgm
|
|
;
|
|
; GFX10-LABEL: buffer_store_bf16:
|
|
; GFX10: ; %bb.0:
|
|
; GFX10-NEXT: buffer_store_short v0, v1, s[0:3], 0 offen
|
|
; GFX10-NEXT: s_endpgm
|
|
;
|
|
; GFX11-LABEL: buffer_store_bf16:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: buffer_store_b16 v0, v1, s[0:3], 0 offen
|
|
; GFX11-NEXT: s_endpgm
|
|
;
|
|
; GFX12-LABEL: buffer_store_bf16:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
|
|
; GFX12-NEXT: buffer_store_b16 v0, v1, s[0:3], null offen
|
|
; GFX12-NEXT: s_endpgm
|
|
call void @llvm.amdgcn.raw.ptr.buffer.store.bf16(bfloat %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_ps void @buffer_store_v2bf16(ptr addrspace(8) inreg %rsrc, <2 x bfloat> %data, i32 %offset) {
|
|
; GFX7-LABEL: buffer_store_v2bf16:
|
|
; GFX7: ; %bb.0:
|
|
; GFX7-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GFX7-NEXT: s_endpgm
|
|
;
|
|
; GFX8-LABEL: buffer_store_v2bf16:
|
|
; GFX8: ; %bb.0:
|
|
; GFX8-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GFX8-NEXT: s_endpgm
|
|
;
|
|
; GFX9-LABEL: buffer_store_v2bf16:
|
|
; GFX9: ; %bb.0:
|
|
; GFX9-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GFX9-NEXT: s_endpgm
|
|
;
|
|
; GFX10-LABEL: buffer_store_v2bf16:
|
|
; GFX10: ; %bb.0:
|
|
; GFX10-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GFX10-NEXT: s_endpgm
|
|
;
|
|
; GFX11-LABEL: buffer_store_v2bf16:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: buffer_store_b32 v0, v1, s[0:3], 0 offen
|
|
; GFX11-NEXT: s_endpgm
|
|
;
|
|
; GFX12-LABEL: buffer_store_v2bf16:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
|
|
; GFX12-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen
|
|
; GFX12-NEXT: s_endpgm
|
|
call void @llvm.amdgcn.raw.ptr.buffer.store.v2bf16(<2 x bfloat> %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_ps void @buffer_store_v4bf16(ptr addrspace(8) inreg %rsrc, <4 x bfloat> %data, i32 %offset) #0 {
|
|
; GFX7-LABEL: buffer_store_v4bf16:
|
|
; GFX7: ; %bb.0:
|
|
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
|
|
; GFX7-NEXT: s_endpgm
|
|
;
|
|
; GFX8-LABEL: buffer_store_v4bf16:
|
|
; GFX8: ; %bb.0:
|
|
; GFX8-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
|
|
; GFX8-NEXT: s_endpgm
|
|
;
|
|
; GFX9-LABEL: buffer_store_v4bf16:
|
|
; GFX9: ; %bb.0:
|
|
; GFX9-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
|
|
; GFX9-NEXT: s_endpgm
|
|
;
|
|
; GFX10-LABEL: buffer_store_v4bf16:
|
|
; GFX10: ; %bb.0:
|
|
; GFX10-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
|
|
; GFX10-NEXT: s_endpgm
|
|
;
|
|
; GFX11-LABEL: buffer_store_v4bf16:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], 0 offen
|
|
; GFX11-NEXT: s_endpgm
|
|
;
|
|
; GFX12-LABEL: buffer_store_v4bf16:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
|
|
; GFX12-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], null offen
|
|
; GFX12-NEXT: s_endpgm
|
|
call void @llvm.amdgcn.raw.ptr.buffer.store.v4bf16(<4 x bfloat> %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
|
|
ret void
|
|
}
|
|
|
|
; FIXME
|
|
; define amdgpu_ps void @buffer_store_v6bf16(ptr addrspace(8) inreg %rsrc, <6 x bfloat> %data, i32 %offset) #0 {
|
|
; call void @llvm.amdgcn.raw.ptr.buffer.store.v6bf16(<6 x bfloat> %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
|
|
; ret void
|
|
; }
|
|
|
|
define amdgpu_ps void @buffer_store_v8bf16(ptr addrspace(8) inreg %rsrc, <8 x bfloat> %data, i32 %offset) #0 {
|
|
; GFX7-LABEL: buffer_store_v8bf16:
|
|
; GFX7: ; %bb.0:
|
|
; GFX7-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 offen
|
|
; GFX7-NEXT: s_endpgm
|
|
;
|
|
; GFX8-LABEL: buffer_store_v8bf16:
|
|
; GFX8: ; %bb.0:
|
|
; GFX8-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 offen
|
|
; GFX8-NEXT: s_endpgm
|
|
;
|
|
; GFX9-LABEL: buffer_store_v8bf16:
|
|
; GFX9: ; %bb.0:
|
|
; GFX9-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 offen
|
|
; GFX9-NEXT: s_endpgm
|
|
;
|
|
; GFX10-LABEL: buffer_store_v8bf16:
|
|
; GFX10: ; %bb.0:
|
|
; GFX10-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 offen
|
|
; GFX10-NEXT: s_endpgm
|
|
;
|
|
; GFX11-LABEL: buffer_store_v8bf16:
|
|
; GFX11: ; %bb.0:
|
|
; GFX11-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], 0 offen
|
|
; GFX11-NEXT: s_endpgm
|
|
;
|
|
; GFX12-LABEL: buffer_store_v8bf16:
|
|
; GFX12: ; %bb.0:
|
|
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
|
|
; GFX12-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null offen
|
|
; GFX12-NEXT: s_endpgm
|
|
call void @llvm.amdgcn.raw.ptr.buffer.store.v8bf16(<8 x bfloat> %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
|
|
ret void
|
|
}
|