b6c0f1bfa7
This is an alternative to #117866 that works by demanding a valid vtype instead of using a separate pass. The main advantage of this is that it allows coalesceVSETVLIs to just reuse an existing vsetvli later in the block. To do this we need to first transfer the vsetvli info to some arbitrary valid state in transferBefore when we encounter a vector copy. Then we add a new vill demanded field that will happily accept any other known vtype, which allows us to coalesce these where possible. Note we also need to check for vector copies in computeVLVTYPEChanges, otherwise the pass will completely skip over functions that only have vector copies and nothing else. This is one part of a fix for #114518. We still need to check if there's other cases where vector copies/whole register moves that are inserted after vsetvli insertion.
256 lines
11 KiB
LLVM
256 lines
11 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh,+zvfbfmin | FileCheck --check-prefixes=CHECK,RV32 %s
|
|
; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfmin | FileCheck --check-prefixes=CHECK,RV64 %s
|
|
; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck --check-prefixes=CHECK,RV32 %s
|
|
; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck --check-prefixes=CHECK,RV64 %s
|
|
|
|
; Integers
|
|
|
|
define void @vector_interleave_store_nxv32i1_nxv16i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv32i1_nxv16i1:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e8, m2, ta, ma
|
|
; CHECK-NEXT: vmv1r.v v9, v0
|
|
; CHECK-NEXT: vmv1r.v v0, v8
|
|
; CHECK-NEXT: vmv.v.i v10, 0
|
|
; CHECK-NEXT: li a1, -1
|
|
; CHECK-NEXT: csrr a2, vlenb
|
|
; CHECK-NEXT: vmerge.vim v12, v10, 1, v0
|
|
; CHECK-NEXT: vmv1r.v v0, v9
|
|
; CHECK-NEXT: vmerge.vim v8, v10, 1, v0
|
|
; CHECK-NEXT: srli a2, a2, 2
|
|
; CHECK-NEXT: vwaddu.vv v16, v8, v12
|
|
; CHECK-NEXT: vwmaccu.vx v16, a1, v12
|
|
; CHECK-NEXT: vmsne.vi v8, v18, 0
|
|
; CHECK-NEXT: vmsne.vi v9, v16, 0
|
|
; CHECK-NEXT: add a1, a2, a2
|
|
; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma
|
|
; CHECK-NEXT: vslideup.vx v9, v8, a2
|
|
; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma
|
|
; CHECK-NEXT: vsm.v v9, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b)
|
|
store <vscale x 32 x i1> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
; Shouldn't be lowered to vsseg because it's unaligned
|
|
define void @vector_interleave_store_nxv16i16_nxv8i16_align1(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv16i16_nxv8i16_align1:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma
|
|
; CHECK-NEXT: vwaddu.vv v12, v8, v10
|
|
; CHECK-NEXT: li a1, -1
|
|
; CHECK-NEXT: vwmaccu.vx v12, a1, v10
|
|
; CHECK-NEXT: vs4r.v v12, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
|
|
store <vscale x 16 x i16> %res, ptr %p, align 1
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv16i16_nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv16i16_nxv8i16:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma
|
|
; CHECK-NEXT: vsseg2e16.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
|
|
store <vscale x 16 x i16> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv8i32_nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv8i32_nxv4i32:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma
|
|
; CHECK-NEXT: vsseg2e32.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
|
|
store <vscale x 8 x i32> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv4i64_nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv4i64_nxv2i64:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e64, m2, ta, ma
|
|
; CHECK-NEXT: vsseg2e64.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b)
|
|
store <vscale x 4 x i64> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv8i64_nxv4i64(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv8i64_nxv4i64:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e64, m4, ta, ma
|
|
; CHECK-NEXT: vsseg2e64.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b)
|
|
store <vscale x 8 x i64> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
; This shouldn't be lowered to a vsseg because EMUL * NFIELDS >= 8
|
|
define void @vector_interleave_store_nxv16i64_nxv8i64(<vscale x 8 x i64> %a, <vscale x 8 x i64> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv16i64_nxv8i64:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: csrr a1, vlenb
|
|
; CHECK-NEXT: vsetvli a2, zero, e16, m2, ta, mu
|
|
; CHECK-NEXT: vid.v v6
|
|
; CHECK-NEXT: vmv8r.v v24, v8
|
|
; CHECK-NEXT: srli a2, a1, 1
|
|
; CHECK-NEXT: vmv4r.v v28, v16
|
|
; CHECK-NEXT: vmv4r.v v16, v12
|
|
; CHECK-NEXT: vsrl.vi v8, v6, 1
|
|
; CHECK-NEXT: vand.vi v10, v6, 1
|
|
; CHECK-NEXT: slli a1, a1, 3
|
|
; CHECK-NEXT: vmsne.vi v0, v10, 0
|
|
; CHECK-NEXT: add a1, a0, a1
|
|
; CHECK-NEXT: vadd.vx v8, v8, a2, v0.t
|
|
; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma
|
|
; CHECK-NEXT: vrgatherei16.vv v0, v24, v8
|
|
; CHECK-NEXT: vrgatherei16.vv v24, v16, v8
|
|
; CHECK-NEXT: vs8r.v v24, (a1)
|
|
; CHECK-NEXT: vs8r.v v0, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 16 x i64> @llvm.vector.interleave2.nxv16i64(<vscale x 8 x i64> %a, <vscale x 8 x i64> %b)
|
|
store <vscale x 16 x i64> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
declare <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
|
|
declare <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16>, <vscale x 8 x i16>)
|
|
declare <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
|
|
declare <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64>, <vscale x 2 x i64>)
|
|
declare <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64>, <vscale x 4 x i64>)
|
|
declare <vscale x 16 x i64> @llvm.vector.interleave2.nxv16i64(<vscale x 8 x i64>, <vscale x 8 x i64>)
|
|
|
|
; Floats
|
|
|
|
define void @vector_interleave_store_nxv4bf16_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv4bf16_nxv2bf16:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma
|
|
; CHECK-NEXT: vsseg2e16.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 4 x bfloat> @llvm.vector.interleave2.nxv4bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
|
|
store <vscale x 4 x bfloat> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv8bf16_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv8bf16_nxv4bf16:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
|
|
; CHECK-NEXT: vsseg2e16.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 8 x bfloat> @llvm.vector.interleave2.nxv8bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
|
|
store <vscale x 8 x bfloat> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv4f16_nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv4f16_nxv2f16:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma
|
|
; CHECK-NEXT: vsseg2e16.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 4 x half> @llvm.vector.interleave2.nxv4f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b)
|
|
store <vscale x 4 x half> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv8f16_nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv8f16_nxv4f16:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
|
|
; CHECK-NEXT: vsseg2e16.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 8 x half> @llvm.vector.interleave2.nxv8f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b)
|
|
store <vscale x 8 x half> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv4f32_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv4f32_nxv2f32:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
|
|
; CHECK-NEXT: vsseg2e32.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 4 x float> @llvm.vector.interleave2.nxv4f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
|
|
store <vscale x 4 x float> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv16bf16_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv16bf16_nxv8bf16:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma
|
|
; CHECK-NEXT: vsseg2e16.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 16 x bfloat> @llvm.vector.interleave2.nxv16bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
|
|
store <vscale x 16 x bfloat> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv16f16_nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv16f16_nxv8f16:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma
|
|
; CHECK-NEXT: vsseg2e16.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 16 x half> @llvm.vector.interleave2.nxv16f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
|
|
store <vscale x 16 x half> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv8f32_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv8f32_nxv4f32:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma
|
|
; CHECK-NEXT: vsseg2e32.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
|
|
store <vscale x 8 x float> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv4f64_nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, ptr %p) {
|
|
; CHECK-LABEL: vector_interleave_store_nxv4f64_nxv2f64:
|
|
; CHECK: # %bb.0:
|
|
; CHECK-NEXT: vsetvli a1, zero, e64, m2, ta, ma
|
|
; CHECK-NEXT: vsseg2e64.v v8, (a0)
|
|
; CHECK-NEXT: ret
|
|
%res = call <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
|
|
store <vscale x 4 x double> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
define void @vector_interleave_store_nxv4p0_nxv2p0(<vscale x 2 x ptr> %a, <vscale x 2 x ptr> %b, ptr %p) {
|
|
; RV32-LABEL: vector_interleave_store_nxv4p0_nxv2p0:
|
|
; RV32: # %bb.0:
|
|
; RV32-NEXT: vsetvli a1, zero, e32, m1, ta, ma
|
|
; RV32-NEXT: vsseg2e32.v v8, (a0)
|
|
; RV32-NEXT: ret
|
|
;
|
|
; RV64-LABEL: vector_interleave_store_nxv4p0_nxv2p0:
|
|
; RV64: # %bb.0:
|
|
; RV64-NEXT: vsetvli a1, zero, e64, m2, ta, ma
|
|
; RV64-NEXT: vsseg2e64.v v8, (a0)
|
|
; RV64-NEXT: ret
|
|
%res = call <vscale x 4 x ptr> @llvm.vector.interleave2.nxv4p0(<vscale x 2 x ptr> %a, <vscale x 2 x ptr> %b)
|
|
store <vscale x 4 x ptr> %res, ptr %p
|
|
ret void
|
|
}
|
|
|
|
declare <vscale x 4 x half> @llvm.vector.interleave2.nxv4f16(<vscale x 2 x half>, <vscale x 2 x half>)
|
|
declare <vscale x 8 x half> @llvm.vector.interleave2.nxv8f16(<vscale x 4 x half>, <vscale x 4 x half>)
|
|
declare <vscale x 4 x float> @llvm.vector.interleave2.nxv4f32(<vscale x 2 x float>, <vscale x 2 x float>)
|
|
declare <vscale x 16 x half> @llvm.vector.interleave2.nxv16f16(<vscale x 8 x half>, <vscale x 8 x half>)
|
|
declare <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float>, <vscale x 4 x float>)
|
|
declare <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double>, <vscale x 2 x double>)
|
|
declare <vscale x 4 x ptr> @llvm.vector.interleave2.nxv4p0(<vscale x 2 x ptr>, <vscale x 2 x ptr>)
|