15b0fabb21
LLVM vector reduction intrinsics return a scalar result, but on RISC-V
vector reduction instructions write the result in the first element of a
vector register. So when a reduction in a loop uses a scalar phi, we end
up with unnecessary scalar moves:
loop:
vfmv.s.f v10, fa0
vfredosum.vs v8, v8, v10
vfmv.f.s fa0, v8
This mainly affects ordered fadd reductions, which has a scalar accumulator
operand.
This tries to vectorize any scalar phis that feed into a fadd reduction
in RISCVCodeGenPrepare, converting:
loop:
%phi = phi <float> [ ..., %entry ], [ %acc, %loop]
%acc = call float @llvm.vector.reduce.fadd.nxv4f32(float %phi, <vscale x 2 x float> %vec)
```
to
loop:
%phi = phi <vscale x 2 x float> [ ..., %entry ], [ %acc.vec, %loop]
%phi.scalar = extractelement <vscale x 2 x float> %phi, i64 0
%acc = call float @llvm.vector.reduce.fadd.nxv4f32(float %x, <vscale x 2 x float> %vec)
%acc.vec = insertelement <vscale x 2 x float> poison, float %acc.next, i64 0
Which eliminates the scalar -> vector -> scalar crossing during
instruction selection.
45 lines
1.6 KiB
LLVM
45 lines
1.6 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
|
|
; RUN: llc < %s -mtriple=riscv64 -mattr=+v | FileCheck %s
|
|
|
|
declare i64 @llvm.vscale.i64()
|
|
declare float @llvm.vector.reduce.fadd.nxv4f32(float, <vscale x 4 x float>)
|
|
|
|
define float @reduce_fadd(ptr %f) {
|
|
; CHECK-LABEL: reduce_fadd:
|
|
; CHECK: # %bb.0: # %entry
|
|
; CHECK-NEXT: csrr a2, vlenb
|
|
; CHECK-NEXT: srli a1, a2, 1
|
|
; CHECK-NEXT: vsetvli a3, zero, e32, m1, ta, ma
|
|
; CHECK-NEXT: vmv.s.x v8, zero
|
|
; CHECK-NEXT: slli a2, a2, 1
|
|
; CHECK-NEXT: li a3, 1024
|
|
; CHECK-NEXT: .LBB0_1: # %vector.body
|
|
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: vl2re32.v v10, (a0)
|
|
; CHECK-NEXT: vsetvli a4, zero, e32, m2, ta, ma
|
|
; CHECK-NEXT: vfredosum.vs v8, v10, v8
|
|
; CHECK-NEXT: sub a3, a3, a1
|
|
; CHECK-NEXT: add a0, a0, a2
|
|
; CHECK-NEXT: bnez a3, .LBB0_1
|
|
; CHECK-NEXT: # %bb.2: # %exit
|
|
; CHECK-NEXT: vfmv.f.s fa0, v8
|
|
; CHECK-NEXT: ret
|
|
entry:
|
|
%vscale = tail call i64 @llvm.vscale.i64()
|
|
%vecsize = shl nuw nsw i64 %vscale, 2
|
|
br label %vector.body
|
|
|
|
vector.body:
|
|
%index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
|
|
%vec.phi = phi float [ 0.000000e+00, %entry ], [ %acc, %vector.body ]
|
|
%gep = getelementptr inbounds float, ptr %f, i64 %index
|
|
%wide.load = load <vscale x 4 x float>, ptr %gep, align 4
|
|
%acc = tail call float @llvm.vector.reduce.fadd.nxv4f32(float %vec.phi, <vscale x 4 x float> %wide.load)
|
|
%index.next = add nuw i64 %index, %vecsize
|
|
%done = icmp eq i64 %index.next, 1024
|
|
br i1 %done, label %exit, label %vector.body
|
|
|
|
exit:
|
|
ret float %acc
|
|
}
|