42526d240c
This commit adds support for casting memrefs into fat raw buffer pointers to the AMDGPU dialect. Fat raw buffer pointers - or, in LLVM terms, ptr addrspcae(7), allow encapsulating a buffer descriptor (as produced by the make.buffer.rsrc intrinsic or provided from some API) into a pointer that supports ordinary pointer operations like load or store. This allows people to take advantage of the additional semantics that buffer_load and similar instructions provide without forcing the use of entirely separate amdgpu.raw_buffer_* operations. Operations on fat raw buffer pointers are translated to the corresponding LLVM intrinsics by the backend. This commit also goes and and defines a #amdgpu.address_space<> attribute so that AMDGPU-specific memory spaces can be represented. Only #amdgpu.address_space<fat_raw_buffer> will work correctly with the memref dialect, but the other possible address spaces are included for completeness. --------- Co-authored-by: Jakub Kuderski <kubakuderski@gmail.com> Co-authored-by: Prashant Kumar <pk5561@gmail.com>
153 lines
8.9 KiB
MLIR
153 lines
8.9 KiB
MLIR
// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s
|
|
// Verify the printed output can be parsed.
|
|
// RUN: mlir-opt -allow-unregistered-dialect %s | mlir-opt -allow-unregistered-dialect | FileCheck %s
|
|
// Verify the generic form can be parsed.
|
|
// RUN: mlir-opt -allow-unregistered-dialect -mlir-print-op-generic %s | mlir-opt -allow-unregistered-dialect | FileCheck %s
|
|
|
|
// CHECK-LABEL: func @ext_packed_fp8
|
|
// CHECK: amdgpu.ext_packed_fp8
|
|
func.func @ext_packed_fp8(%v: vector<4xf8E4M3FNUZ>) -> f32 {
|
|
%ret = amdgpu.ext_packed_fp8 %v[0] : vector<4xf8E4M3FNUZ> to f32
|
|
func.return %ret : f32
|
|
}
|
|
|
|
// CHECK-LABEL: func @packed_trunc_2xfp8
|
|
// CHECK: amdgpu.packed_trunc_2xfp8
|
|
func.func @packed_trunc_2xfp8(%v1: f32, %v2: f32, %others: vector<4xf8E5M2FNUZ>, %stoch: i32) -> vector<4xf8E5M2FNUZ> {
|
|
%ret = amdgpu.packed_trunc_2xfp8 %v1, %v2 into %others[word 1] : f32 to vector<4xf8E5M2FNUZ> into vector<4xf8E5M2FNUZ>
|
|
func.return %ret : vector<4xf8E5M2FNUZ>
|
|
}
|
|
|
|
// CHECK-LABEL: func @packed_stoch_round_fp8
|
|
// CHECK: amdgpu.packed_stoch_round_fp8
|
|
func.func @packed_stoch_round_fp8(%v1: f32, %stoch: i32, %others: vector<4xf8E5M2FNUZ>) -> vector<4xf8E5M2FNUZ> {
|
|
%ret = amdgpu.packed_stoch_round_fp8 %v1 + %stoch into %others[2] : f32 to vector<4xf8E5M2FNUZ> into vector<4xf8E5M2FNUZ>
|
|
func.return %ret : vector<4xf8E5M2FNUZ>
|
|
}
|
|
|
|
// CHECK-LABEL: func @fat_raw_buffer_cast_easy
|
|
// CHECK: amdgpu.fat_raw_buffer_cast
|
|
func.func @fat_raw_buffer_cast_easy(%m: memref<8xi32>) -> memref<8xi32, #amdgpu.address_space<fat_raw_buffer>> {
|
|
%ret = amdgpu.fat_raw_buffer_cast %m : memref<8xi32> to memref<8xi32, #amdgpu.address_space<fat_raw_buffer>>
|
|
func.return %ret : memref<8xi32, #amdgpu.address_space<fat_raw_buffer>>
|
|
}
|
|
|
|
// CHECK-LABEL: func @fat_raw_buffer_cast
|
|
// CHECK: amdgpu.fat_raw_buffer_cast
|
|
// CHECK-SAME: validBytes(%{{[^)]*}})
|
|
// CHECK-SAME: cacheSwizzleStride(%{{[^)]*}})
|
|
// CHECK-SAME: boundsCheck(false)
|
|
// CHECK-SAME: resetOffset
|
|
func.func @fat_raw_buffer_cast(%m: memref<8xi32, strided<[1], offset: ?>>, %validBytes: i32, %cacheSwizzle: i14) -> memref<8xi32, strided<[1]>, #amdgpu.address_space<fat_raw_buffer>> {
|
|
%ret = amdgpu.fat_raw_buffer_cast %m validBytes(%validBytes) cacheSwizzleStride(%cacheSwizzle) boundsCheck(false) resetOffset
|
|
: memref<8xi32, strided<[1], offset: ?>> to memref<8xi32, strided<[1]>, #amdgpu.address_space<fat_raw_buffer>>
|
|
func.return %ret : memref<8xi32, strided<[1]>, #amdgpu.address_space<fat_raw_buffer>>
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_load_f32_from_rank_1
|
|
func.func @raw_buffer_load_f32_from_rank_1(%src : memref<128xf32>, %offset : i32, %idx0 : i32) -> f32 {
|
|
// CHECK: amdgpu.raw_buffer_load {indexOffset = 1 : i32} %{{.*}}[{{.*}}] sgprOffset %{{.*}} : memref<128xf32>, i32 -> f32
|
|
%0 = amdgpu.raw_buffer_load {boundsCheck = true, indexOffset = 1 : i32} %src[%idx0] sgprOffset %offset : memref<128xf32>, i32 -> f32
|
|
func.return %0 : f32
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_load_f32_from_rank_4
|
|
func.func @raw_buffer_load_f32_from_rank_4(%src : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) -> f32 {
|
|
// CHECK: amdgpu.raw_buffer_load {indexOffset = 1 : i32} %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : memref<128x64x32x16xf32>, i32, i32, i32, i32 -> f32
|
|
%0 = amdgpu.raw_buffer_load {boundsCheck = true, indexOffset = 1 : i32} %src[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : memref<128x64x32x16xf32>, i32, i32, i32, i32 -> f32
|
|
func.return %0 : f32
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_load_scalar
|
|
func.func @raw_buffer_load_scalar(%src : memref<f32>) -> f32 {
|
|
// CHECK: amdgpu.raw_buffer_load {indexOffset = 1 : i32} %{{.*}}[] : memref<f32> -> f32
|
|
%0 = amdgpu.raw_buffer_load {indexOffset = 1 : i32} %src[] : memref<f32> -> f32
|
|
func.return %0 : f32
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_load_4xf32_from_rank_4
|
|
func.func @raw_buffer_load_4xf32_from_rank_4(%src : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) -> vector<4xf32> {
|
|
// CHECK: amdgpu.raw_buffer_load {indexOffset = 1 : i32} %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : memref<128x64x32x16xf32>, i32, i32, i32, i32 -> vector<4xf32>
|
|
%0 = amdgpu.raw_buffer_load {boundsCheck = true, indexOffset = 1 : i32} %src[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : memref<128x64x32x16xf32>, i32, i32, i32, i32 -> vector<4xf32>
|
|
func.return %0 : vector<4xf32>
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_store_f32_to_rank_1
|
|
func.func @raw_buffer_store_f32_to_rank_1(%value : f32, %dst : memref<128xf32>, %offset : i32, %idx0 : i32) {
|
|
// CHECK: amdgpu.raw_buffer_store {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128xf32>, i32
|
|
amdgpu.raw_buffer_store {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0] sgprOffset %offset : f32 -> memref<128xf32>, i32
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_store_f32_to_rank_4
|
|
func.func @raw_buffer_store_f32_to_rank_4(%value : f32, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {
|
|
// CHECK: amdgpu.raw_buffer_store {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32
|
|
amdgpu.raw_buffer_store {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_store_4xf32_to_rank_4
|
|
func.func @raw_buffer_store_4xf32_to_rank_4(%value : vector<4xf32>, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {
|
|
// CHECK: amdgpu.raw_buffer_store {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : vector<4xf32> -> memref<128x64x32x16xf32>, i32, i32, i32, i32
|
|
amdgpu.raw_buffer_store {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : vector<4xf32> -> memref<128x64x32x16xf32>, i32, i32, i32, i32
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_store_scalar
|
|
func.func @raw_buffer_store_scalar(%value : f32, %dst : memref<f32>) {
|
|
// CHECK: amdgpu.raw_buffer_store {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[] : f32 -> memref<f32>
|
|
amdgpu.raw_buffer_store {indexOffset = 1 : i32} %value -> %dst[] : f32 -> memref<f32>
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_atomic_fadd_f32_to_rank_1
|
|
func.func @raw_buffer_atomic_fadd_f32_to_rank_1(%value : f32, %dst : memref<128xf32>, %offset : i32, %idx0 : i32) {
|
|
// CHECK: amdgpu.raw_buffer_atomic_fadd {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128xf32>, i32
|
|
amdgpu.raw_buffer_atomic_fadd {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0] sgprOffset %offset : f32 -> memref<128xf32>, i32
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_atomic_fadd_f32_to_rank_4
|
|
func.func @raw_buffer_atomic_fadd_f32_to_rank_4(%value : f32, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {
|
|
// CHECK: amdgpu.raw_buffer_atomic_fadd {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32
|
|
amdgpu.raw_buffer_atomic_fadd {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @raw_buffer_atomic_cmpswap_f32
|
|
func.func @raw_buffer_atomic_cmpswap_f32(%src : f32, %cmp : f32, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {
|
|
// CHECK: amdgpu.raw_buffer_atomic_cmpswap {indexOffset = 1 : i32} %{{.*}}, %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32
|
|
amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true, indexOffset = 1 : i32} %src, %cmp -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @lds_barrier
|
|
func.func @lds_barrier() {
|
|
// CHECK: amdgpu.lds_barrier
|
|
amdgpu.lds_barrier
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @sched_barrier
|
|
func.func @sched_barrier() {
|
|
// CHECK: amdgpu.sched_barrier allow = <none>
|
|
amdgpu.sched_barrier allow = <none>
|
|
// CHECK: amdgpu.sched_barrier allow = <valu|all_vmem>
|
|
amdgpu.sched_barrier allow = <valu|all_vmem>
|
|
func.return
|
|
}
|
|
|
|
// CHECK-LABEL: func @mfma
|
|
func.func @mfma(%arg0 : f32, %arg1 : vector<32xf32>) -> vector<32xf32> {
|
|
// CHECK: amdgpu.mfma
|
|
%0 = amdgpu.mfma %arg0 * %arg0 + %arg1 { abid = 1 : i32, cbsz = 1 : i32, k = 1 : i32, m = 32 : i32, n = 32 : i32, blocks = 2 : i32 } blgp = bcast_second_32 : f32, f32, vector<32xf32>
|
|
func.return %0 : vector<32xf32>
|
|
}
|
|
|
|
// CHECK-LABEL: func @wmma
|
|
func.func @wmma(%arg0 : vector<16xf16>, %arg1 : vector<8xf16>) -> vector<8xf16> {
|
|
// CHECK: amdgpu.wmma
|
|
%0 = amdgpu.wmma %arg0 * %arg0 + %arg1 : vector<16xf16>, vector<16xf16>, vector<8xf16>
|
|
func.return %0 : vector<8xf16>
|
|
}
|