llvm-project/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-contents-legalization.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s

; Note: if you're adding tests here, also add them to
; buffer-fat-pointers-contents-legalization.ll to make sure the output of this
; transformation can codegen.

target triple = "amdgcn--"

;;; Legal types. These are natively supported, no casts should be performed.

define i8 @load_i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i8 @load_i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret i8 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i8, ptr addrspace(7) %p
  ret i8 %ret
}

define void @store_i8(i8 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i8(
; CHECK-SAME: i8 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i8 %data, ptr addrspace(7) %p
  ret void
}

define i16 @load_i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i16 @load_i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret i16 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i16, ptr addrspace(7) %p
  ret i16 %ret
}

define void @store_i16(i16 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i16(
; CHECK-SAME: i16 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i16 %data, ptr addrspace(7) %p
  ret void
}

define i32 @load_i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i32 @load_i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret i32 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i32, ptr addrspace(7) %p
  ret i32 %ret
}

define void @store_i32(i32 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i32(
; CHECK-SAME: i32 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i32 %data, ptr addrspace(7) %p
  ret void
}

define i64 @load_i64(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i64 @load_i64(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret i64 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i64, ptr addrspace(7) %p
  ret i64 %ret
}

define void @store_i64(i64 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i64(
; CHECK-SAME: i64 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i64 %data, ptr addrspace(7) %p
  ret void
}

define i128 @load_i128(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i128 @load_i128(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call i128 @llvm.amdgcn.raw.ptr.buffer.load.i128(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret i128 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i128, ptr addrspace(7) %p
  ret i128 %ret
}

define void @store_i128(i128 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i128(
; CHECK-SAME: i128 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i128(i128 [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i128 %data, ptr addrspace(7) %p
  ret void
}

define <1 x i32> @load_v1i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <1 x i32> @load_v1i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <1 x i32>
; CHECK-NEXT:    ret <1 x i32> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <1 x i32>, ptr addrspace(7) %p
  ret <1 x i32> %ret
}

define void @store_v1i32(<1 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i32(
; CHECK-SAME: <1 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <1 x i32> [[DATA]] to i32
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <1 x i32> %data, ptr addrspace(7) %p
  ret void
}

define <2 x i32> @load_v2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i32> @load_v2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <2 x i32> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x i32>, ptr addrspace(7) %p
  ret <2 x i32> %ret
}

define void @store_v2i32(<2 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i32(
; CHECK-SAME: <2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x i32> %data, ptr addrspace(7) %p
  ret void
}

define <3 x i32> @load_v3i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x i32> @load_v3i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <3 x i32> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <3 x i32>, ptr addrspace(7) %p
  ret <3 x i32> %ret
}

define void @store_v3i32(<3 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3i32(
; CHECK-SAME: <3 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <3 x i32> %data, ptr addrspace(7) %p
  ret void
}

define <4 x i32> @load_v4i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i32> @load_v4i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <4 x i32> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <4 x i32>, ptr addrspace(7) %p
  ret <4 x i32> %ret
}

define void @store_v4i32(<4 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i32(
; CHECK-SAME: <4 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <4 x i32> %data, ptr addrspace(7) %p
  ret void
}

define <2 x i16> @load_v2i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i16> @load_v2i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <2 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x i16>, ptr addrspace(7) %p
  ret <2 x i16> %ret
}

define void @store_v2i16(<2 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i16(
; CHECK-SAME: <2 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x i16> %data, ptr addrspace(7) %p
  ret void
}

define <4 x i16> @load_v4i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i16> @load_v4i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <4 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <4 x i16>, ptr addrspace(7) %p
  ret <4 x i16> %ret
}

define void @store_v4i16(<4 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i16(
; CHECK-SAME: <4 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <4 x i16> %data, ptr addrspace(7) %p
  ret void
}

define <8 x i16> @load_v8i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x i16> @load_v8i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <8 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <8 x i16>, ptr addrspace(7) %p
  ret <8 x i16> %ret
}

define void @store_v8i16(<8 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i16(
; CHECK-SAME: <8 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <8 x i16> %data, ptr addrspace(7) %p
  ret void
}

define <2 x i64> @load_v2i64(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i64> @load_v2i64(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <2 x i64> @llvm.amdgcn.raw.ptr.buffer.load.v2i64(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <2 x i64> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x i64>, ptr addrspace(7) %p
  ret <2 x i64> %ret
}

define void @store_v2i64(<2 x i64> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i64(
; CHECK-SAME: <2 x i64> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i64(<2 x i64> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x i64> %data, ptr addrspace(7) %p
  ret void
}

define half @load_f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define half @load_f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret half [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load half, ptr addrspace(7) %p
  ret half %ret
}

define void @store_f16(half %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_f16(
; CHECK-SAME: half [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store half %data, ptr addrspace(7) %p
  ret void
}

define bfloat @load_bf16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define bfloat @load_bf16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call bfloat @llvm.amdgcn.raw.ptr.buffer.load.bf16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret bfloat [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load bfloat, ptr addrspace(7) %p
  ret bfloat %ret
}

define void @store_bf16(bfloat %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_bf16(
; CHECK-SAME: bfloat [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.bf16(bfloat [[DATA]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store bfloat %data, ptr addrspace(7) %p
  ret void
}

define <2 x half> @load_v2f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x half> @load_v2f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <2 x half> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x half>, ptr addrspace(7) %p
  ret <2 x half> %ret
}

define void @store_v2f16(<2 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2f16(
; CHECK-SAME: <2 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x half> %data, ptr addrspace(7) %p
  ret void
}

define <4 x bfloat> @load_v4bf16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x bfloat> @load_v4bf16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <4 x bfloat> @llvm.amdgcn.raw.ptr.buffer.load.v4bf16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <4 x bfloat> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <4 x bfloat>, ptr addrspace(7) %p
  ret <4 x bfloat> %ret
}

define void @store_v4bf16(<4 x bfloat> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4bf16(
; CHECK-SAME: <4 x bfloat> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4bf16(<4 x bfloat> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <4 x bfloat> %data, ptr addrspace(7) %p
  ret void
}

define <8 x half> @load_v8f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x half> @load_v8f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <8 x half> @llvm.amdgcn.raw.ptr.buffer.load.v8f16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <8 x half> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <8 x half>, ptr addrspace(7) %p
  ret <8 x half> %ret
}

define void @store_v8f16(<8 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8f16(
; CHECK-SAME: <8 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v8f16(<8 x half> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <8 x half> %data, ptr addrspace(7) %p
  ret void
}

define float @load_f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define float @load_f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret float [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load float, ptr addrspace(7) %p
  ret float %ret
}

define void @store_f32(float %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_f32(
; CHECK-SAME: float [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store float %data, ptr addrspace(7) %p
  ret void
}

define <2 x float> @load_v2f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x float> @load_v2f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <2 x float> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x float>, ptr addrspace(7) %p
  ret <2 x float> %ret
}

define void @store_v2f32(<2 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2f32(
; CHECK-SAME: <2 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x float> %data, ptr addrspace(7) %p
  ret void
}

define <3 x float> @load_v3f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x float> @load_v3f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <3 x float> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <3 x float>, ptr addrspace(7) %p
  ret <3 x float> %ret
}

define void @store_v3f32(<3 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3f32(
; CHECK-SAME: <3 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <3 x float> %data, ptr addrspace(7) %p
  ret void
}

define <4 x float> @load_v4f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x float> @load_v4f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <4 x float> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <4 x float>, ptr addrspace(7) %p
  ret <4 x float> %ret
}

define void @store_v4f32(<4 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4f32(
; CHECK-SAME: <4 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <4 x float> %data, ptr addrspace(7) %p
  ret void
}

define ptr addrspace(0) @load_p0(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr @load_p0(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call ptr @llvm.amdgcn.raw.ptr.buffer.load.p0(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret ptr [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load ptr addrspace(0), ptr addrspace(7) %p
  ret ptr addrspace(0) %ret
}

define void @store_p0(ptr addrspace(0) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p0(
; CHECK-SAME: ptr [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p0(ptr [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store ptr addrspace(0) %data, ptr addrspace(7) %p
  ret void
}

define ptr addrspace(1) @load_p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(1) @load_p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(1) @llvm.amdgcn.raw.ptr.buffer.load.p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret ptr addrspace(1) [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load ptr addrspace(1), ptr addrspace(7) %p
  ret ptr addrspace(1) %ret
}

define void @store_p1(ptr addrspace(1) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p1(
; CHECK-SAME: ptr addrspace(1) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p1(ptr addrspace(1) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store ptr addrspace(1) %data, ptr addrspace(7) %p
  ret void
}

define ptr addrspace(2) @load_p2(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(2) @load_p2(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(2) @llvm.amdgcn.raw.ptr.buffer.load.p2(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret ptr addrspace(2) [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load ptr addrspace(2), ptr addrspace(7) %p
  ret ptr addrspace(2) %ret
}

define void @store_p2(ptr addrspace(2) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p2(
; CHECK-SAME: ptr addrspace(2) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p2(ptr addrspace(2) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store ptr addrspace(2) %data, ptr addrspace(7) %p
  ret void
}

define ptr addrspace(3) @load_p3(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(3) @load_p3(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(3) @llvm.amdgcn.raw.ptr.buffer.load.p3(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret ptr addrspace(3) [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load ptr addrspace(3), ptr addrspace(7) %p
  ret ptr addrspace(3) %ret
}

define void @store_p3(ptr addrspace(3) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p3(
; CHECK-SAME: ptr addrspace(3) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p3(ptr addrspace(3) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store ptr addrspace(3) %data, ptr addrspace(7) %p
  ret void
}

define ptr addrspace(4) @load_p4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(4) @load_p4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(4) @llvm.amdgcn.raw.ptr.buffer.load.p4(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret ptr addrspace(4) [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load ptr addrspace(4), ptr addrspace(7) %p
  ret ptr addrspace(4) %ret
}

define void @store_p4(ptr addrspace(4) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p4(
; CHECK-SAME: ptr addrspace(4) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p4(ptr addrspace(4) [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store ptr addrspace(4) %data, ptr addrspace(7) %p
  ret void
}

define ptr addrspace(5) @load_p5(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(5) @load_p5(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(5) @llvm.amdgcn.raw.ptr.buffer.load.p5(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret ptr addrspace(5) [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load ptr addrspace(5), ptr addrspace(7) %p
  ret ptr addrspace(5) %ret
}

define void @store_p5(ptr addrspace(5) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p5(
; CHECK-SAME: ptr addrspace(5) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p5(ptr addrspace(5) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store ptr addrspace(5) %data, ptr addrspace(7) %p
  ret void
}

define ptr addrspace(6) @load_p6(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(6) @load_p6(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(6) @llvm.amdgcn.raw.ptr.buffer.load.p6(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret ptr addrspace(6) [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load ptr addrspace(6), ptr addrspace(7) %p
  ret ptr addrspace(6) %ret
}

define void @store_p6(ptr addrspace(6) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p6(
; CHECK-SAME: ptr addrspace(6) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p6(ptr addrspace(6) [[DATA]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store ptr addrspace(6) %data, ptr addrspace(7) %p
  ret void
}

define ptr addrspace(8) @load_p8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define ptr addrspace(8) @load_p8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call ptr addrspace(8) @llvm.amdgcn.raw.ptr.buffer.load.p8(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret ptr addrspace(8) [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load ptr addrspace(8), ptr addrspace(7) %p
  ret ptr addrspace(8) %ret
}

define void @store_p8(ptr addrspace(8) %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_p8(
; CHECK-SAME: ptr addrspace(8) [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.p8(ptr addrspace(8) [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store ptr addrspace(8) %data, ptr addrspace(7) %p
  ret void
}

define <2 x ptr addrspace(1)> @load_v2p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x ptr addrspace(1)> @load_v2p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <2 x ptr addrspace(1)> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x ptr addrspace(1)>, ptr addrspace(7) %p
  ret <2 x ptr addrspace(1)> %ret
}

define void @store_v2p1(<2 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2p1(
; CHECK-SAME: <2 x ptr addrspace(1)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x ptr addrspace(1)> %data, ptr addrspace(7) %p
  ret void
}

define <2 x ptr addrspace(5)> @load_v2p5(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x ptr addrspace(5)> @load_v2p5(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <2 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v2p5(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <2 x ptr addrspace(5)> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x ptr addrspace(5)>, ptr addrspace(7) %p
  ret <2 x ptr addrspace(5)> %ret
}

define void @store_v2p5(<2 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2p5(
; CHECK-SAME: <2 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p5(<2 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x ptr addrspace(5)> %data, ptr addrspace(7) %p
  ret void
}

define <3 x ptr addrspace(5)> @load_v3p5(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x ptr addrspace(5)> @load_v3p5(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <3 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v3p5(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <3 x ptr addrspace(5)> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <3 x ptr addrspace(5)>, ptr addrspace(7) %p
  ret <3 x ptr addrspace(5)> %ret
}

define void @store_v3p5(<3 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3p5(
; CHECK-SAME: <3 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3p5(<3 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <3 x ptr addrspace(5)> %data, ptr addrspace(7) %p
  ret void
}

define <4 x ptr addrspace(5)> @load_v4p5(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x ptr addrspace(5)> @load_v4p5(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <4 x ptr addrspace(5)> @llvm.amdgcn.raw.ptr.buffer.load.v4p5(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <4 x ptr addrspace(5)> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <4 x ptr addrspace(5)>, ptr addrspace(7) %p
  ret <4 x ptr addrspace(5)> %ret
}

define void @store_v4p5(<4 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4p5(
; CHECK-SAME: <4 x ptr addrspace(5)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4p5(<4 x ptr addrspace(5)> [[DATA]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <4 x ptr addrspace(5)> %data, ptr addrspace(7) %p
  ret void
}

;;; 3 words in a short type. These need to be bitcast to <3 x i32> to be supported.

define <6 x half> @load_v6f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x half> @load_v6f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <6 x half>
; CHECK-NEXT:    ret <6 x half> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <6 x half>, ptr addrspace(7) %p
  ret <6 x half> %ret
}

define void @store_v6f16(<6 x half> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6f16(
; CHECK-SAME: <6 x half> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <6 x half> [[DATA]] to <3 x i32>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <6 x half> %data, ptr addrspace(7) %p
  ret void
}

;;; Long types (32 bit elements). Must be split into multiple operations.

define <5 x float> @load_v5f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <5 x float> @load_v5f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x float> poison, <5 x float> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertelement <5 x float> [[RET_PARTS_0]], float [[RET_OFF_16]], i64 4
; CHECK-NEXT:    ret <5 x float> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <5 x float>, ptr addrspace(7) %p
  ret <5 x float> %ret
}

define void @store_v5f32(<5 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v5f32(
; CHECK-SAME: <5 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x float> [[DATA]], <5 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x float> [[DATA]], i64 4
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <5 x float> %data, ptr addrspace(7) %p
  ret void
}

define <6 x float> @load_v6f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x float> @load_v6f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <6 x float> poison, <6 x float> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <2 x float> [[RET_OFF_16]], <2 x float> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET:%.*]] = shufflevector <6 x float> [[RET_PARTS_0]], <6 x float> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
; CHECK-NEXT:    ret <6 x float> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <6 x float>, ptr addrspace(7) %p
  ret <6 x float> %ret
}

define void @store_v6f32(<6 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6f32(
; CHECK-SAME: <6 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <6 x float> [[DATA]], <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <6 x float> [[DATA]], <6 x float> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <6 x float> %data, ptr addrspace(7) %p
  ret void
}

define <7 x float> @load_v7f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <7 x float> @load_v7f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <7 x float> poison, <7 x float> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 4, i32 5, i32 6>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <3 x float> @llvm.amdgcn.raw.ptr.buffer.load.v3f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <3 x float> [[RET_OFF_16]], <3 x float> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET:%.*]] = shufflevector <7 x float> [[RET_PARTS_0]], <7 x float> [[RET_EXT_4]], <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 8, i32 9>
; CHECK-NEXT:    ret <7 x float> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <7 x float>, ptr addrspace(7) %p
  ret <7 x float> %ret
}

define void @store_v7f32(<7 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v7f32(
; CHECK-SAME: <7 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <7 x float> [[DATA]], <7 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <7 x float> [[DATA]], <7 x float> poison, <3 x i32> <i32 4, i32 5, i32 6>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3f32(<3 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <7 x float> %data, ptr addrspace(7) %p
  ret void
}

define <8 x float> @load_v8f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x float> @load_v8f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <8 x float> poison, <8 x float> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <4 x float> [[RET_OFF_16]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET:%.*]] = shufflevector <8 x float> [[RET_PARTS_0]], <8 x float> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT:    ret <8 x float> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <8 x float>, ptr addrspace(7) %p
  ret <8 x float> %ret
}

define void @store_v8f32(<8 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8f32(
; CHECK-SAME: <8 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <8 x float> [[DATA]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <8 x float> [[DATA]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <8 x float> %data, ptr addrspace(7) %p
  ret void
}

define <10 x float> @load_v10f32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <10 x float> @load_v10f32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x float> [[RET_OFF_0]], <4 x float> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <10 x float> poison, <10 x float> [[RET_EXT_0]], <10 x i32> <i32 10, i32 11, i32 12, i32 13, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <4 x float> [[RET_OFF_16]], <4 x float> poison, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <10 x float> [[RET_PARTS_0]], <10 x float> [[RET_EXT_4]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 10, i32 11, i32 12, i32 13, i32 8, i32 9>
; CHECK-NEXT:    [[RET_OFF_32:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_8:%.*]] = shufflevector <2 x float> [[RET_OFF_32]], <2 x float> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET:%.*]] = shufflevector <10 x float> [[RET_PARTS_4]], <10 x float> [[RET_EXT_8]], <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
; CHECK-NEXT:    ret <10 x float> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <10 x float>, ptr addrspace(7) %p
  ret <10 x float> %ret
}

define void @store_v10f32(<10 x float> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v10f32(
; CHECK-SAME: <10 x float> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_0]], ptr addrspace(8) align 64 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_8:%.*]] = shufflevector <10 x float> [[DATA]], <10 x float> poison, <2 x i32> <i32 8, i32 9>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f32(<2 x float> [[DATA_SLICE_8]], ptr addrspace(8) align 32 [[BUF]], i32 32, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <10 x float> %data, ptr addrspace(7) %p
  ret void
}

define <6 x i32> @load_v6i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i32> @load_v6i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <6 x i32> poison, <6 x i32> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET:%.*]] = shufflevector <6 x i32> [[RET_PARTS_0]], <6 x i32> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
; CHECK-NEXT:    ret <6 x i32> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <6 x i32>, ptr addrspace(7) %p
  ret <6 x i32> %ret
}

define void @store_v6i32(<6 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6i32(
; CHECK-SAME: <6 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <6 x i32> %data, ptr addrspace(7) %p
  ret void
}

define <4 x ptr addrspace(1)> @load_v4p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x ptr addrspace(1)> @load_v4p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x ptr addrspace(1)> [[RET_OFF_0]], <2 x ptr addrspace(1)> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <4 x ptr addrspace(1)> poison, <4 x ptr addrspace(1)> [[RET_EXT_0]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_2:%.*]] = shufflevector <2 x ptr addrspace(1)> [[RET_OFF_16]], <2 x ptr addrspace(1)> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET:%.*]] = shufflevector <4 x ptr addrspace(1)> [[RET_PARTS_0]], <4 x ptr addrspace(1)> [[RET_EXT_2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; CHECK-NEXT:    ret <4 x ptr addrspace(1)> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <4 x ptr addrspace(1)>, ptr addrspace(7) %p
  ret <4 x ptr addrspace(1)> %ret
}

define void @store_v4p1(<4 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4p1(
; CHECK-SAME: <4 x ptr addrspace(1)> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <4 x ptr addrspace(1)> [[DATA]], <4 x ptr addrspace(1)> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = shufflevector <4 x ptr addrspace(1)> [[DATA]], <4 x ptr addrspace(1)> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_SLICE_2]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <4 x ptr addrspace(1)> %data, ptr addrspace(7) %p
  ret void
}

;;; Uneven types with 16-bit elements. Require splitting into multiple operations.

define <1 x i16> @load_v1i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <1 x i16> @load_v1i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <1 x i16>
; CHECK-NEXT:    ret <1 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <1 x i16>, ptr addrspace(7) %p
  ret <1 x i16> %ret
}

define void @store_v1i16(<1 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i16(
; CHECK-SAME: <1 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <1 x i16> [[DATA]] to i16
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_STORABLE]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <1 x i16> %data, ptr addrspace(7) %p
  ret void
}

define <3 x i16> @load_v3i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x i16> @load_v3i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 2
; CHECK-NEXT:    ret <3 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <3 x i16>, ptr addrspace(7) %p
  ret <3 x i16> %ret
}

define void @store_v3i16(<3 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3i16(
; CHECK-SAME: <3 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA]], i64 2
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <3 x i16> %data, ptr addrspace(7) %p
  ret void
}

define <5 x i16> @load_v5i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <5 x i16> @load_v5i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v4i16(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i16> [[RET_OFF_0]], <4 x i16> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x i16> poison, <5 x i16> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
; CHECK-NEXT:    [[RET_OFF_8:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertelement <5 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_8]], i64 4
; CHECK-NEXT:    ret <5 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <5 x i16>, ptr addrspace(7) %p
  ret <5 x i16> %ret
}

define void @store_v5i16(<5 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v5i16(
; CHECK-SAME: <5 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x i16> [[DATA]], <5 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i16(<4 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x i16> [[DATA]], i64 4
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <5 x i16> %data, ptr addrspace(7) %p
  ret void
}

define <6 x i16> @load_v6i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i16> @load_v6i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <6 x i16>
; CHECK-NEXT:    ret <6 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <6 x i16>, ptr addrspace(7) %p
  ret <6 x i16> %ret
}

define void @store_v6i16(<6 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6i16(
; CHECK-SAME: <6 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <6 x i16> [[DATA]] to <3 x i32>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <6 x i16> %data, ptr addrspace(7) %p
  ret void
}

define <7 x i16> @load_v7i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <7 x i16> @load_v7i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast <3 x i32> [[RET_OFF_0]] to <6 x i16>
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <6 x i16> [[RET_OFF_0_FROM_LOADABLE]], <6 x i16> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <7 x i16> poison, <7 x i16> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 6>
; CHECK-NEXT:    [[RET_OFF_12:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertelement <7 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_12]], i64 6
; CHECK-NEXT:    ret <7 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <7 x i16>, ptr addrspace(7) %p
  ret <7 x i16> %ret
}

define void @store_v7i16(<7 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v7i16(
; CHECK-SAME: <7 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <7 x i16> [[DATA]], <7 x i16> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <6 x i16> [[DATA_SLICE_0]] to <3 x i32>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_6:%.*]] = extractelement <7 x i16> [[DATA]], i64 6
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_6]], ptr addrspace(8) align 4 [[BUF]], i32 12, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <7 x i16> %data, ptr addrspace(7) %p
  ret void
}

define <9 x i16> @load_v9i16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <9 x i16> @load_v9i16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <8 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v8i16(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <8 x i16> [[RET_OFF_0]], <8 x i16> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <9 x i16> poison, <9 x i16> [[RET_EXT_0]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertelement <9 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_16]], i64 8
; CHECK-NEXT:    ret <9 x i16> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <9 x i16>, ptr addrspace(7) %p
  ret <9 x i16> %ret
}

define void @store_v9i16(<9 x i16> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v9i16(
; CHECK-SAME: <9 x i16> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <9 x i16> [[DATA]], <9 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v8i16(<8 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_8:%.*]] = extractelement <9 x i16> [[DATA]], i64 8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_8]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <9 x i16> %data, ptr addrspace(7) %p
  ret void
}

;;; Byte vectors. Need to be
;;; - Split into multiple operations
;;; - Bitcast if they have a natively supported width

define <1 x i8> @load_v1i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <1 x i8> @load_v1i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i8 [[RET_LOADABLE]] to <1 x i8>
; CHECK-NEXT:    ret <1 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <1 x i8>, ptr addrspace(7) %p
  ret <1 x i8> %ret
}

define void @store_v1i8(<1 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v1i8(
; CHECK-SAME: <1 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <1 x i8> [[DATA]] to i8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <1 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <2 x i8> @load_v2i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i8> @load_v2i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <2 x i8>
; CHECK-NEXT:    ret <2 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x i8>, ptr addrspace(7) %p
  ret <2 x i8> %ret
}

define void @store_v2i8(<2 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i8(
; CHECK-SAME: <2 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <2 x i8> [[DATA]] to i16
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <3 x i8> @load_v3i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <3 x i8> @load_v3i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i16 [[RET_OFF_0]] to <2 x i8>
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x i8> [[RET_OFF_0_FROM_LOADABLE]], <2 x i8> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <3 x i8> poison, <3 x i8> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
; CHECK-NEXT:    [[RET_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertelement <3 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_2]], i64 2
; CHECK-NEXT:    ret <3 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <3 x i8>, ptr addrspace(7) %p
  ret <3 x i8> %ret
}

define void @store_v3i8(<3 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v3i8(
; CHECK-SAME: <3 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <3 x i8> [[DATA]], <3 x i8> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <2 x i8> [[DATA_SLICE_0]] to i16
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = extractelement <3 x i8> [[DATA]], i64 2
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_2]], ptr addrspace(8) align 2 [[BUF]], i32 2, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <3 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <4 x i8> @load_v4i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i8> @load_v4i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
; CHECK-NEXT:    ret <4 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <4 x i8>, ptr addrspace(7) %p
  ret <4 x i8> %ret
}

define void @store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i8(
; CHECK-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <4 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <5 x i8> @load_v5i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <5 x i8> @load_v5i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x i8> poison, <5 x i8> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertelement <5 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_4]], i64 4
; CHECK-NEXT:    ret <5 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <5 x i8>, ptr addrspace(7) %p
  ret <5 x i8> %ret
}

define void @store_v5i8(<5 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v5i8(
; CHECK-SAME: <5 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x i8> [[DATA]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x i8> [[DATA]], i64 4
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <5 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <6 x i8> @load_v6i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i8> @load_v6i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    [[RET_SLICE_2:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 2
; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i16> [[RET_SLICE_2]] to <6 x i8>
; CHECK-NEXT:    ret <6 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <6 x i8>, ptr addrspace(7) %p
  ret <6 x i8> %ret
}

define void @store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v6i8(
; CHECK-SAME: <6 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <6 x i8> [[DATA]] to <3 x i16>
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA_LEGAL]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA_LEGAL]], i64 2
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <6 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <7 x i8> @load_v7i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <7 x i8> @load_v7i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <7 x i8> poison, <7 x i8> [[RET_EXT_0]], <7 x i32> <i32 7, i32 8, i32 9, i32 10, i32 4, i32 5, i32 6>
; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    [[RET_OFF_4_FROM_LOADABLE:%.*]] = bitcast i16 [[RET_OFF_4]] to <2 x i8>
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <2 x i8> [[RET_OFF_4_FROM_LOADABLE]], <2 x i8> poison, <7 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <7 x i8> [[RET_PARTS_0]], <7 x i8> [[RET_EXT_4]], <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 8, i32 6>
; CHECK-NEXT:    [[RET_OFF_6:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertelement <7 x i8> [[RET_PARTS_4]], i8 [[RET_OFF_6]], i64 6
; CHECK-NEXT:    ret <7 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <7 x i8>, ptr addrspace(7) %p
  ret <7 x i8> %ret
}

define void @store_v7i8(<7 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v7i8(
; CHECK-SAME: <7 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <7 x i8> [[DATA]], <7 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <7 x i8> [[DATA]], <7 x i8> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT:    [[DATA_SLICE_4_STORABLE:%.*]] = bitcast <2 x i8> [[DATA_SLICE_4]] to i16
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_4_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_6:%.*]] = extractelement <7 x i8> [[DATA]], i64 6
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_6]], ptr addrspace(8) align 2 [[BUF]], i32 6, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <7 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <8 x i8> @load_v8i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x i8> @load_v8i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast <2 x i32> [[RET_LOADABLE]] to <8 x i8>
; CHECK-NEXT:    ret <8 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <8 x i8>, ptr addrspace(7) %p
  ret <8 x i8> %ret
}

define void @store_v8i8(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i8(
; CHECK-SAME: <8 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <8 x i8> [[DATA]] to <2 x i32>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <8 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <12 x i8> @load_v12i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <12 x i8> @load_v12i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to <12 x i8>
; CHECK-NEXT:    ret <12 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <12 x i8>, ptr addrspace(7) %p
  ret <12 x i8> %ret
}

define void @store_v12i8(<12 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v12i8(
; CHECK-SAME: <12 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <12 x i8> [[DATA]] to <3 x i32>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <12 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <16 x i8> @load_v16i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <16 x i8> @load_v16i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast <4 x i32> [[RET_LOADABLE]] to <16 x i8>
; CHECK-NEXT:    ret <16 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <16 x i8>, ptr addrspace(7) %p
  ret <16 x i8> %ret
}

define void @store_v16i8(<16 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v16i8(
; CHECK-SAME: <16 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <16 x i8> [[DATA]] to <4 x i32>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 16 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <16 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <32 x i8> @load_v32i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <32 x i8> @load_v32i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT:    [[RET:%.*]] = bitcast <8 x i32> [[RET_PARTS_4]] to <32 x i8>
; CHECK-NEXT:    ret <32 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <32 x i8>, ptr addrspace(7) %p
  ret <32 x i8> %ret
}

define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v32i8(
; CHECK-SAME: <32 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <32 x i8> [[DATA]] to <8 x i32>
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <32 x i8> %data, ptr addrspace(7) %p
  ret void
}

;;; Arrays. Need to become vectors.

define [1 x i32] @load_a1i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [1 x i32] @load_a1i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_LOADABLE]] to <1 x i32>
; CHECK-NEXT:    [[RET_ELEM_0:%.*]] = extractelement <1 x i32> [[RET_FROM_LOADABLE]], i64 0
; CHECK-NEXT:    [[RET:%.*]] = insertvalue [1 x i32] poison, i32 [[RET_ELEM_0]], 0
; CHECK-NEXT:    ret [1 x i32] [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load [1 x i32], ptr addrspace(7) %p
  ret [1 x i32] %ret
}

define void @store_a1i32([1 x i32] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a1i32(
; CHECK-SAME: [1 x i32] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_ELEM_0:%.*]] = extractvalue [1 x i32] [[DATA]], 0
; CHECK-NEXT:    [[DATA_AS_VEC_0:%.*]] = insertelement <1 x i32> poison, i32 [[DATA_ELEM_0]], i64 0
; CHECK-NEXT:    [[DATA_STORABLE:%.*]] = bitcast <1 x i32> [[DATA_AS_VEC_0]] to i32
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_STORABLE]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store [1 x i32] %data, ptr addrspace(7) %p
  ret void
}

define [2 x i32] @load_a2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x i32] @load_a2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_ELEM_0:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 0
; CHECK-NEXT:    [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET_ELEM_0]], 0
; CHECK-NEXT:    [[RET_ELEM_1:%.*]] = extractelement <2 x i32> [[RET_LOADABLE]], i64 1
; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x i32] [[RET_AS_ARRAY_0]], i32 [[RET_ELEM_1]], 1
; CHECK-NEXT:    ret [2 x i32] [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load [2 x i32], ptr addrspace(7) %p
  ret [2 x i32] %ret
}

define void @store_a2i32([2 x i32] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2i32(
; CHECK-SAME: [2 x i32] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_ELEM_0:%.*]] = extractvalue [2 x i32] [[DATA]], 0
; CHECK-NEXT:    [[DATA_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA_ELEM_0]], i64 0
; CHECK-NEXT:    [[DATA_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA]], 1
; CHECK-NEXT:    [[DATA_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA_AS_VEC_0]], i32 [[DATA_ELEM_1]], i64 1
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store [2 x i32] %data, ptr addrspace(7) %p
  ret void
}

define [2 x half] @load_a2f16(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x half] @load_a2f16(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.v2f16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_ELEM_0:%.*]] = extractelement <2 x half> [[RET_LOADABLE]], i64 0
; CHECK-NEXT:    [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x half] poison, half [[RET_ELEM_0]], 0
; CHECK-NEXT:    [[RET_ELEM_1:%.*]] = extractelement <2 x half> [[RET_LOADABLE]], i64 1
; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x half] [[RET_AS_ARRAY_0]], half [[RET_ELEM_1]], 1
; CHECK-NEXT:    ret [2 x half] [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load [2 x half], ptr addrspace(7) %p
  ret [2 x half] %ret
}

define void @store_a2f16([2 x half] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2f16(
; CHECK-SAME: [2 x half] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_ELEM_0:%.*]] = extractvalue [2 x half] [[DATA]], 0
; CHECK-NEXT:    [[DATA_AS_VEC_0:%.*]] = insertelement <2 x half> poison, half [[DATA_ELEM_0]], i64 0
; CHECK-NEXT:    [[DATA_ELEM_1:%.*]] = extractvalue [2 x half] [[DATA]], 1
; CHECK-NEXT:    [[DATA_AS_VEC_1:%.*]] = insertelement <2 x half> [[DATA_AS_VEC_0]], half [[DATA_ELEM_1]], i64 1
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2f16(<2 x half> [[DATA_AS_VEC_1]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store [2 x half] %data, ptr addrspace(7) %p
  ret void
}

define [2 x ptr addrspace(1)] @load_a2p1(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x ptr addrspace(1)] @load_a2p1(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <2 x ptr addrspace(1)> @llvm.amdgcn.raw.ptr.buffer.load.v2p1(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_ELEM_0:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 0
; CHECK-NEXT:    [[RET_AS_ARRAY_0:%.*]] = insertvalue [2 x ptr addrspace(1)] poison, ptr addrspace(1) [[RET_ELEM_0]], 0
; CHECK-NEXT:    [[RET_ELEM_1:%.*]] = extractelement <2 x ptr addrspace(1)> [[RET_LOADABLE]], i64 1
; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x ptr addrspace(1)] [[RET_AS_ARRAY_0]], ptr addrspace(1) [[RET_ELEM_1]], 1
; CHECK-NEXT:    ret [2 x ptr addrspace(1)] [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load [2 x ptr addrspace(1)], ptr addrspace(7) %p
  ret [2 x ptr addrspace(1)] %ret
}

define void @store_a2p1([2 x ptr addrspace(1)] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2p1(
; CHECK-SAME: [2 x ptr addrspace(1)] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_ELEM_0:%.*]] = extractvalue [2 x ptr addrspace(1)] [[DATA]], 0
; CHECK-NEXT:    [[DATA_AS_VEC_0:%.*]] = insertelement <2 x ptr addrspace(1)> poison, ptr addrspace(1) [[DATA_ELEM_0]], i64 0
; CHECK-NEXT:    [[DATA_ELEM_1:%.*]] = extractvalue [2 x ptr addrspace(1)] [[DATA]], 1
; CHECK-NEXT:    [[DATA_AS_VEC_1:%.*]] = insertelement <2 x ptr addrspace(1)> [[DATA_AS_VEC_0]], ptr addrspace(1) [[DATA_ELEM_1]], i64 1
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2p1(<2 x ptr addrspace(1)> [[DATA_AS_VEC_1]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store [2 x ptr addrspace(1)] %data, ptr addrspace(7) %p
  ret void
}

;;; Scalars of atypical width. Need to be cast to vectors and split.

define i40 @load_i40(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i40 @load_i40(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_OFF_0_FROM_LOADABLE:%.*]] = bitcast i32 [[RET_OFF_0]] to <4 x i8>
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i8> [[RET_OFF_0_FROM_LOADABLE]], <4 x i8> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x i8> poison, <5 x i8> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    [[RET_SLICE_4:%.*]] = insertelement <5 x i8> [[RET_PARTS_0]], i8 [[RET_OFF_4]], i64 4
; CHECK-NEXT:    [[RET:%.*]] = bitcast <5 x i8> [[RET_SLICE_4]] to i40
; CHECK-NEXT:    ret i40 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i40, ptr addrspace(7) %p
  ret i40 %ret
}

define void @store_i40(i40 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i40(
; CHECK-SAME: i40 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast i40 [[DATA]] to <5 x i8>
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x i8> [[DATA_LEGAL]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    [[DATA_SLICE_0_STORABLE:%.*]] = bitcast <4 x i8> [[DATA_SLICE_0]] to i32
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_0_STORABLE]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x i8> [[DATA_LEGAL]], i64 4
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_SLICE_4]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i40 %data, ptr addrspace(7) %p
  ret void
}

define i96 @load_i96(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i96 @load_i96(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call <3 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v3i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i32> [[RET_LOADABLE]] to i96
; CHECK-NEXT:    ret i96 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i96, ptr addrspace(7) %p
  ret i96 %ret
}

define void @store_i96(i96 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i96(
; CHECK-SAME: i96 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast i96 [[DATA]] to <3 x i32>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> [[DATA_LEGAL]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i96 %data, ptr addrspace(7) %p
  ret void
}

define i160 @load_i160(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i160 @load_i160(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <5 x i32> poison, <5 x i32> [[RET_EXT_0]], <5 x i32> <i32 5, i32 6, i32 7, i32 8, i32 4>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_SLICE_4:%.*]] = insertelement <5 x i32> [[RET_PARTS_0]], i32 [[RET_OFF_16]], i64 4
; CHECK-NEXT:    [[RET:%.*]] = bitcast <5 x i32> [[RET_SLICE_4]] to i160
; CHECK-NEXT:    ret i160 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i160, ptr addrspace(7) %p
  ret i160 %ret
}

define void @store_i160(i160 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i160(
; CHECK-SAME: i160 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast i160 [[DATA]] to <5 x i32>
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <5 x i32> [[DATA_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = extractelement <5 x i32> [[DATA_LEGAL]], i64 4
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i160 %data, ptr addrspace(7) %p
  ret void
}

define i256 @load_i256(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i256 @load_i256(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[RET_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <4 x i32> [[RET_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <8 x i32> [[RET_PARTS_0]], <8 x i32> [[RET_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT:    [[RET:%.*]] = bitcast <8 x i32> [[RET_PARTS_4]] to i256
; CHECK-NEXT:    ret i256 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i256, ptr addrspace(7) %p
  ret i256 %ret
}

define void @store_i256(i256 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i256(
; CHECK-SAME: i256 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast i256 [[DATA]] to <8 x i32>
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <8 x i32> [[DATA_LEGAL]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i256 %data, ptr addrspace(7) %p
  ret void
}

;;; Non-byte-sized scalars. Require zero-extension.

define i7 @load_i7(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i7 @load_i7(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = trunc i8 [[RET_LOADABLE]] to i7
; CHECK-NEXT:    ret i7 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i7, ptr addrspace(7) %p
  ret i7 %ret
}

define void @store_i7(i7 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i7(
; CHECK-SAME: i7 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_ZEXT:%.*]] = zext i7 [[DATA]] to i8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i7 %data, ptr addrspace(7) %p
  ret void
}

define i4 @load_i4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define i4 @load_i4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = trunc i8 [[RET_LOADABLE]] to i4
; CHECK-NEXT:    ret i4 [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load i4, ptr addrspace(7) %p
  ret i4 %ret
}

define void @store_i4(i4 %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_i4(
; CHECK-SAME: i4 [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_ZEXT:%.*]] = zext i4 [[DATA]] to i8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store i4 %data, ptr addrspace(7) %p
  ret void
}

;;; Byte-sized vectors of i4. Require casts.

define <2 x i4> @load_v2i4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i4> @load_v2i4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i8 [[RET_LOADABLE]] to <2 x i4>
; CHECK-NEXT:    ret <2 x i4> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x i4>, ptr addrspace(7) %p
  ret <2 x i4> %ret
}

define void @store_v2i4(<2 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i4(
; CHECK-SAME: <2 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <2 x i4> [[DATA]] to i8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA_LEGAL]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x i4> %data, ptr addrspace(7) %p
  ret void
}

define <4 x i4> @load_v4i4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i4> @load_v4i4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i16 [[RET_LOADABLE]] to <4 x i4>
; CHECK-NEXT:    ret <4 x i4> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <4 x i4>, ptr addrspace(7) %p
  ret <4 x i4> %ret
}

define void @store_v4i4(<4 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v4i4(
; CHECK-SAME: <4 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <4 x i4> [[DATA]] to i16
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_LEGAL]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <4 x i4> %data, ptr addrspace(7) %p
  ret void
}

define <8 x i4> @load_v8i4(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <8 x i4> @load_v8i4(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <8 x i4>
; CHECK-NEXT:    ret <8 x i4> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <8 x i4>, ptr addrspace(7) %p
  ret <8 x i4> %ret
}

define void @store_v8i4(<8 x i4> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v8i4(
; CHECK-SAME: <8 x i4> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <8 x i4> [[DATA]] to i32
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <8 x i4> %data, ptr addrspace(7) %p
  ret void
}

;;; Vectors of non-byte-sized integers.

define <2 x i6> @load_v2i6(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <2 x i6> @load_v2i6(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_TRUNC:%.*]] = trunc i16 [[RET_LOADABLE]] to i12
; CHECK-NEXT:    [[RET:%.*]] = bitcast i12 [[RET_TRUNC]] to <2 x i6>
; CHECK-NEXT:    ret <2 x i6> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <2 x i6>, ptr addrspace(7) %p
  ret <2 x i6> %ret
}

define void @store_v2i6(<2 x i6> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v2i6(
; CHECK-SAME: <2 x i6> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_AS_SCALAR:%.*]] = bitcast <2 x i6> [[DATA]] to i12
; CHECK-NEXT:    [[DATA_ZEXT:%.*]] = zext i12 [[DATA_AS_SCALAR]] to i16
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_ZEXT]], ptr addrspace(8) align 2 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <2 x i6> %data, ptr addrspace(7) %p
  ret void
}

;; Blocks of fp6 elements
define <6 x i32> @load_v32i6(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i32> @load_v32i6(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <4 x i32> [[RET_OFF_0]], <4 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <6 x i32> poison, <6 x i32> [[RET_EXT_0]], <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 4, i32 5>
; CHECK-NEXT:    [[RET_OFF_16:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    [[RET_EXT_4:%.*]] = shufflevector <2 x i32> [[RET_OFF_16]], <2 x i32> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_4:%.*]] = shufflevector <6 x i32> [[RET_PARTS_0]], <6 x i32> [[RET_EXT_4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
; CHECK-NEXT:    [[RET:%.*]] = bitcast <6 x i32> [[RET_PARTS_4]] to <32 x i6>
; CHECK-NEXT:    [[RET_CAST:%.*]] = bitcast <32 x i6> [[RET]] to <6 x i32>
; CHECK-NEXT:    ret <6 x i32> [[RET_CAST]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <32 x i6>, ptr addrspace(7) %p
  %ret.cast = bitcast <32 x i6> %ret to <6 x i32>
  ret <6 x i32> %ret.cast
}

define void @store_v32i6(<6 x i32> %data.abi, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_v32i6(
; CHECK-SAME: <6 x i32> [[DATA_ABI:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA:%.*]] = bitcast <6 x i32> [[DATA_ABI]] to <32 x i6>
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[DATA_SLICE_0]], ptr addrspace(8) align 32 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_SLICE_4:%.*]] = shufflevector <6 x i32> [[DATA_ABI]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_SLICE_4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %data = bitcast <6 x i32> %data.abi to <32 x i6>
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <32 x i6> %data, ptr addrspace(7) %p
  ret void
}

;;; Modifiers

define <4 x i8> @volatile_load_v4i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <4 x i8> @volatile_load_v4i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_LOADABLE:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648)
; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[RET_LOADABLE]] to <4 x i8>
; CHECK-NEXT:    ret <4 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load volatile <4 x i8>, ptr addrspace(7) %p
  ret <4 x i8> %ret
}

define void @volatile_store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @volatile_store_v4i8(
; CHECK-SAME: <4 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <4 x i8> [[DATA]] to i32
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_LEGAL]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 -2147483648)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store volatile <4 x i8> %data, ptr addrspace(7) %p
  ret void
}

define <6 x i8> @volatile_load_v6i8(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <6 x i8> @volatile_load_v6i8(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_OFF_0:%.*]] = call <2 x i16> @llvm.amdgcn.raw.ptr.buffer.load.v2i16(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648)
; CHECK-NEXT:    [[RET_EXT_0:%.*]] = shufflevector <2 x i16> [[RET_OFF_0]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
; CHECK-NEXT:    [[RET_PARTS_0:%.*]] = shufflevector <3 x i16> poison, <3 x i16> [[RET_EXT_0]], <3 x i32> <i32 3, i32 4, i32 2>
; CHECK-NEXT:    [[RET_OFF_4:%.*]] = call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648)
; CHECK-NEXT:    [[RET_SLICE_2:%.*]] = insertelement <3 x i16> [[RET_PARTS_0]], i16 [[RET_OFF_4]], i64 2
; CHECK-NEXT:    [[RET:%.*]] = bitcast <3 x i16> [[RET_SLICE_2]] to <6 x i8>
; CHECK-NEXT:    ret <6 x i8> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load volatile <6 x i8>, ptr addrspace(7) %p
  ret <6 x i8> %ret
}

define void @volatile_store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @volatile_store_v6i8(
; CHECK-SAME: <6 x i8> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_LEGAL:%.*]] = bitcast <6 x i8> [[DATA]] to <3 x i16>
; CHECK-NEXT:    [[DATA_SLICE_0:%.*]] = shufflevector <3 x i16> [[DATA_LEGAL]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i16(<2 x i16> [[DATA_SLICE_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 -2147483648)
; CHECK-NEXT:    [[DATA_SLICE_2:%.*]] = extractelement <3 x i16> [[DATA_LEGAL]], i64 2
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 [[DATA_SLICE_2]], ptr addrspace(8) align 4 [[BUF]], i32 4, i32 0, i32 -2147483648)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store volatile <6 x i8> %data, ptr addrspace(7) %p
  ret void
}

define [2 x [2 x i32]] @load_a2a2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x [2 x i32]] @load_a2a2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET0_ELEM_0:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 0
; CHECK-NEXT:    [[RET0_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET0_ELEM_0]], 0
; CHECK-NEXT:    [[RET0_ELEM_1:%.*]] = extractelement <2 x i32> [[RET0_OFF_0]], i64 1
; CHECK-NEXT:    [[RET0_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET0_AS_ARRAY_0]], i32 [[RET0_ELEM_1]], 1
; CHECK-NEXT:    [[RET0:%.*]] = insertvalue [2 x [2 x i32]] poison, [2 x i32] [[RET0_AS_ARRAY_1]], 0
; CHECK-NEXT:    [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0)
; CHECK-NEXT:    [[RET1_ELEM_0:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 0
; CHECK-NEXT:    [[RET1_AS_ARRAY_0:%.*]] = insertvalue [2 x i32] poison, i32 [[RET1_ELEM_0]], 0
; CHECK-NEXT:    [[RET1_ELEM_1:%.*]] = extractelement <2 x i32> [[RET1_OFF_8]], i64 1
; CHECK-NEXT:    [[RET1_AS_ARRAY_1:%.*]] = insertvalue [2 x i32] [[RET1_AS_ARRAY_0]], i32 [[RET1_ELEM_1]], 1
; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x [2 x i32]] [[RET0]], [2 x i32] [[RET1_AS_ARRAY_1]], 1
; CHECK-NEXT:    ret [2 x [2 x i32]] [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load [2 x [2 x i32]], ptr addrspace(7) %p
  ret [2 x [2 x i32]] %ret
}

define void @store_a2a2i32([2 x [2 x i32]] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2a2i32(
; CHECK-SAME: [2 x [2 x i32]] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA0:%.*]] = extractvalue [2 x [2 x i32]] [[DATA]], 0
; CHECK-NEXT:    [[DATA0_ELEM_0:%.*]] = extractvalue [2 x i32] [[DATA0]], 0
; CHECK-NEXT:    [[DATA0_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA0_ELEM_0]], i64 0
; CHECK-NEXT:    [[DATA0_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA0]], 1
; CHECK-NEXT:    [[DATA0_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA0_AS_VEC_0]], i32 [[DATA0_ELEM_1]], i64 1
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA1:%.*]] = extractvalue [2 x [2 x i32]] [[DATA]], 1
; CHECK-NEXT:    [[DATA1_ELEM_0:%.*]] = extractvalue [2 x i32] [[DATA1]], 0
; CHECK-NEXT:    [[DATA1_AS_VEC_0:%.*]] = insertelement <2 x i32> poison, i32 [[DATA1_ELEM_0]], i64 0
; CHECK-NEXT:    [[DATA1_ELEM_1:%.*]] = extractvalue [2 x i32] [[DATA1]], 1
; CHECK-NEXT:    [[DATA1_AS_VEC_1:%.*]] = insertelement <2 x i32> [[DATA1_AS_VEC_0]], i32 [[DATA1_ELEM_1]], i64 1
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1_AS_VEC_1]], ptr addrspace(8) align 4 [[BUF]], i32 8, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store [2 x [2 x i32]] %data, ptr addrspace(7) %p
  ret void
}

define [2 x <2 x i32>] @load_a2v2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [2 x <2 x i32>] @load_a2v2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET0:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[RET0_OFF_0]], 0
; CHECK-NEXT:    [[RET1_OFF_8:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertvalue [2 x <2 x i32>] [[RET0]], <2 x i32> [[RET1_OFF_8]], 1
; CHECK-NEXT:    ret [2 x <2 x i32>] [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load [2 x <2 x i32>], ptr addrspace(7) %p
  ret [2 x <2 x i32>] %ret
}

define void @store_a2v2i32([2 x <2 x i32>] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a2v2i32(
; CHECK-SAME: [2 x <2 x i32>] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA0:%.*]] = extractvalue [2 x <2 x i32>] [[DATA]], 0
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA1:%.*]] = extractvalue [2 x <2 x i32>] [[DATA]], 1
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store [2 x <2 x i32>] %data, ptr addrspace(7) %p
  ret void
}

define { i32 } @load_sl_i32s(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define { i32 } @load_sl_i32s(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_0_OFF_0:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i32 } poison, i32 [[RET_0_OFF_0]], 0
; CHECK-NEXT:    ret { i32 } [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load { i32 }, ptr addrspace(7) %p
  ret { i32 } %ret
}

define void @store_sl_i32s({ i32 } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_i32s(
; CHECK-SAME: { i32 } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_0:%.*]] = extractvalue { i32 } [[DATA]], 0
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store { i32 } %data, ptr addrspace(7) %p
  ret void
}

define { { float } } @load_sl_sl_f32ss(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define { { float } } @load_sl_sl_f32ss(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_0_0_OFF_0:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertvalue { { float } } poison, float [[RET_0_0_OFF_0]], 0, 0
; CHECK-NEXT:    ret { { float } } [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load { { float } }, ptr addrspace(7) %p
  ret { { float } } %ret
}

define void @store_sl_sl_f32ss({ { float } } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_sl_f32ss(
; CHECK-SAME: { { float } } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_0_0:%.*]] = extractvalue { { float } } [[DATA]], 0, 0
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[DATA_0_0]], ptr addrspace(8) align 4 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store { { float } } %data, ptr addrspace(7) %p
  ret void
}

define { <2 x i32> } @load_sl_v2i32s(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define { <2 x i32> } @load_sl_v2i32s(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_0_OFF_0:%.*]] = call <2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertvalue { <2 x i32> } poison, <2 x i32> [[RET_0_OFF_0]], 0
; CHECK-NEXT:    ret { <2 x i32> } [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load { <2 x i32> }, ptr addrspace(7) %p
  ret { <2 x i32> } %ret
}

define void @store_sl_v2i32s({ <2 x i32> } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_v2i32s(
; CHECK-SAME: { <2 x i32> } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_0:%.*]] = extractvalue { <2 x i32> } [[DATA]], 0
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v2i32(<2 x i32> [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store { <2 x i32> } %data, ptr addrspace(7) %p
  ret void
}

define { i64, i32 } @load_sl_i64i32s(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define { i64, i32 } @load_sl_i64i32s(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET_0_OFF_0:%.*]] = call i64 @llvm.amdgcn.raw.ptr.buffer.load.i64(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET_0:%.*]] = insertvalue { i64, i32 } poison, i64 [[RET_0_OFF_0]], 0
; CHECK-NEXT:    [[RET_1_OFF_8:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i64, i32 } [[RET_0]], i32 [[RET_1_OFF_8]], 1
; CHECK-NEXT:    ret { i64, i32 } [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load { i64, i32 }, ptr addrspace(7) %p
  ret { i64, i32 } %ret
}

define void @store_sl_i64i32s({ i64, i32 } %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_sl_i64i32s(
; CHECK-SAME: { i64, i32 } [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA_0:%.*]] = extractvalue { i64, i32 } [[DATA]], 0
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 [[DATA_0]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA_1:%.*]] = extractvalue { i64, i32 } [[DATA]], 1
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[DATA_1]], ptr addrspace(8) align 8 [[BUF]], i32 8, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store { i64, i32 } %data, ptr addrspace(7) %p
  ret void
}

define [4 x i7] @load_a4i7(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define [4 x i7] @load_a4i7(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET0_OFF_0:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[RET0_TRUNC:%.*]] = trunc i8 [[RET0_OFF_0]] to i7
; CHECK-NEXT:    [[RET0:%.*]] = insertvalue [4 x i7] poison, i7 [[RET0_TRUNC]], 0
; CHECK-NEXT:    [[RET1_OFF_1:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
; CHECK-NEXT:    [[RET1_TRUNC:%.*]] = trunc i8 [[RET1_OFF_1]] to i7
; CHECK-NEXT:    [[RET1:%.*]] = insertvalue [4 x i7] [[RET0]], i7 [[RET1_TRUNC]], 1
; CHECK-NEXT:    [[RET2_OFF_2:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
; CHECK-NEXT:    [[RET2_TRUNC:%.*]] = trunc i8 [[RET2_OFF_2]] to i7
; CHECK-NEXT:    [[RET2:%.*]] = insertvalue [4 x i7] [[RET1]], i7 [[RET2_TRUNC]], 2
; CHECK-NEXT:    [[RET3_OFF_3:%.*]] = call i8 @llvm.amdgcn.raw.ptr.buffer.load.i8(ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
; CHECK-NEXT:    [[RET3_TRUNC:%.*]] = trunc i8 [[RET3_OFF_3]] to i7
; CHECK-NEXT:    [[RET:%.*]] = insertvalue [4 x i7] [[RET2]], i7 [[RET3_TRUNC]], 3
; CHECK-NEXT:    ret [4 x i7] [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load [4 x i7], ptr addrspace(7) %p
  ret [4 x i7] %ret
}

define void @store_a4i7([4 x i7] %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_a4i7(
; CHECK-SAME: [4 x i7] [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[DATA0:%.*]] = extractvalue [4 x i7] [[DATA]], 0
; CHECK-NEXT:    [[DATA0_ZEXT:%.*]] = zext i7 [[DATA0]] to i8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA0_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    [[DATA1:%.*]] = extractvalue [4 x i7] [[DATA]], 1
; CHECK-NEXT:    [[DATA1_ZEXT:%.*]] = zext i7 [[DATA1]] to i8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA1_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 1, i32 0, i32 0)
; CHECK-NEXT:    [[DATA2:%.*]] = extractvalue [4 x i7] [[DATA]], 2
; CHECK-NEXT:    [[DATA2_ZEXT:%.*]] = zext i7 [[DATA2]] to i8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA2_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 2, i32 0, i32 0)
; CHECK-NEXT:    [[DATA3:%.*]] = extractvalue [4 x i7] [[DATA]], 3
; CHECK-NEXT:    [[DATA3_ZEXT:%.*]] = zext i7 [[DATA3]] to i8
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i8(i8 [[DATA3_ZEXT]], ptr addrspace(8) align 1 [[BUF]], i32 3, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store [4 x i7] %data, ptr addrspace(7) %p
  ret void
}

;;; Scalable vector. This isn't semantically meaningful but shouldn't crash.

define <vscale x 2 x i32> @load_nxv2i32(ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define <vscale x 2 x i32> @load_nxv2i32(
; CHECK-SAME: ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[RET:%.*]] = call <vscale x 2 x i32> @llvm.amdgcn.raw.ptr.buffer.load.nxv2i32(ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret <vscale x 2 x i32> [[RET]]
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  %ret = load <vscale x 2 x i32>, ptr addrspace(7) %p
  ret <vscale x 2 x i32> %ret
}

define void @store_nxv2i32(<vscale x 2 x i32> %data, ptr addrspace(8) inreg %buf) {
; CHECK-LABEL: define void @store_nxv2i32(
; CHECK-SAME: <vscale x 2 x i32> [[DATA:%.*]], ptr addrspace(8) inreg [[BUF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.nxv2i32(<vscale x 2 x i32> [[DATA]], ptr addrspace(8) align 8 [[BUF]], i32 0, i32 0, i32 0)
; CHECK-NEXT:    ret void
;
  %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
  store <vscale x 2 x i32> %data, ptr addrspace(7) %p
  ret void
}