shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
Luke Lau 825129378e
[VPlan] Move tail folding out of VPlanPredicator. NFC (#176143) 
Currently the logic for introducing a header mask and predicating the
vector loop region is done inside introduceMasksAndLinearize.

This splits the tail folding part out into an individual VPlan transform
so that VPlanPredicator.cpp doesn't need to worry about tail folding,
which seemed to be a temporary measure according to a comment in
VPlanTransforms.h.

To perform tail folding independently, this splits the "body" of the
vector loop region between the phis in the header and the branch + iv
increment in the latch:

Before:

```
+-------------------------------------------+
|%iv = ...                                  |
|...                                        |
|%iv.next = add %iv, vfxuf                  |
|branch-on-count %iv.next, vector-trip-count|
+-------------------------------------------+
```

After:
```
+-------------------------------------------+
|%iv = ...                                  |
|%wide.iv = widen-canonical-iv ...          |
|%header-mask = icmp ule %wide.iv, BTC      |---+
|branch-on-cond %header-mask                |   |
+-------------------------------------------+   |
                     |                          |
                     v                          |
+-------------------------------------------+   |
|...                                        |   |
+-------------------------------------------+   |
                     |                          |
                     v                          |
+-------------------------------------------+   |
|%iv.next = add %iv, vfxuf                  |<--+
|branch-on-count %iv.next, vector-trip-count|
+-------------------------------------------+
```

Phis are then inserted in the latch for any value in the loop body that
have outside uses, with poison as their incoming value from the header
edge.

The motivation for this is to allow us to share the same "predicate all
successor blocks" type of predication we do for tail folding, but for
early-exit loops in #172454. This may also allow us to directly emit an
EVL based header mask, instead of having to match + transform the
existing header mask in addExplicitVectorLength.

This also allows us to eventually handle recurrences in the same
transform, avoiding the need to special case tail folding in
addReductionResultComputation.
2026-03-05 08:17:37 +00:00

339 lines
14 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S -vplan-print-after=foldTailByMasking -disable-output 2>&1 | FileCheck %s
define i32 @live_out(ptr noalias %p, i32 %n) {
; CHECK-LABEL: VPlan for loop in 'live_out'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1>
; CHECK-NEXT: EMIT store ir<%y>, ir<%gep>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<%y>, vector.body.split ], [ ir<poison>, vector.body ]
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-lane vp<[[VP10]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane vp<[[VP12]]>, vp<[[VP8]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: IR %y.lcssa = phi i32 [ %y, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP15:%[0-9]+]]> = phi [ ir<%iv>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[VP15]]> from scalar.ph)
; CHECK-NEXT: IR %gep = getelementptr i32, ptr %p, i32 %iv
; CHECK-NEXT: IR %x = load i32, ptr %gep, align 4
; CHECK-NEXT: IR %y = add i32 %x, 1
; CHECK-NEXT: IR store i32 %y, ptr %gep, align 4
; CHECK-NEXT: IR %iv.next = add i32 %iv, 1
; CHECK-NEXT: IR %ec = icmp eq i32 %iv.next, %n
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop
loop:
%iv = phi i32 [0, %entry], [%iv.next, %loop]
%gep = getelementptr i32, ptr %p, i32 %iv
%x = load i32, ptr %gep
%y = add i32 %x, 1
store i32 %y, ptr %gep
%iv.next = add i32 %iv, 1
%ec = icmp eq i32 %iv.next, %n
br i1 %ec, label %exit, label %loop
exit:
ret i32 %y
}
define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-LABEL: VPlan for loop in 'conditional_live_out'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
; CHECK-NEXT: EMIT branch-on-cond ir<%c>
; CHECK-NEXT: Successor(s): if, latch
; CHECK-EMPTY:
; CHECK-NEXT: if:
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1>
; CHECK-NEXT: EMIT store ir<%y>, ir<%gep>
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
; CHECK-NEXT: EMIT-SCALAR ir<%phi> = phi [ ir<%y>, if ], [ ir<0>, vector.body.split ]
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<%phi>, latch ], [ ir<poison>, vector.body ]
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-lane vp<[[VP10]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane vp<[[VP12]]>, vp<[[VP8]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: IR %phi.lcssa = phi i32 [ %phi, %latch ] (extra operand: vp<[[VP13]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP15:%[0-9]+]]> = phi [ ir<%iv>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<[[VP15]]> from scalar.ph)
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop
loop:
%iv = phi i32 [0, %entry], [%iv.next, %latch]
br i1 %c, label %if, label %latch
if:
%gep = getelementptr i32, ptr %p, i32 %iv
%x = load i32, ptr %gep
%y = add i32 %x, 1
store i32 %y, ptr %gep
br label %latch
latch:
%phi = phi i32 [0, %loop], [%y, %if]
%iv.next = add i32 %iv, 1
%ec = icmp eq i32 %iv.next, %n
br i1 %ec, label %exit, label %loop
exit:
ret i32 %phi
}
define void @header_unconditional_branch(ptr noalias %p, i32 %n) {
; CHECK-LABEL: VPlan for loop in 'header_unconditional_branch'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = phi [ ir<%iv>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<[[VP10]]> from scalar.ph)
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop
loop:
%iv = phi i32 [0, %entry], [%iv.next, %latch]
br label %latch
latch:
%iv.next = add i32 %iv, 1
%ec = icmp eq i32 %iv.next, %n
br i1 %ec, label %exit, label %loop
exit:
ret void
}
define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-LABEL: VPlan for loop in 'reduction'
; CHECK: VPlan ' for UF>=1' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi ir<0>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%rdx.next> = add ir<%rdx>, ir<%x>
; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP8]]> = phi [ ir<%rdx.next>, vector.body.split ], [ ir<poison>, vector.body ]
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-lane vp<[[VP10]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane vp<[[VP12]]>, vp<[[VP8]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
; CHECK-NEXT: IR %rdx.next.lcssa = phi i32 [ %rdx.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
; CHECK-NEXT: EMIT-SCALAR vp<[[VP15:%[0-9]+]]> = phi [ ir<%iv>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: EMIT-SCALAR vp<[[VP16:%[0-9]+]]> = phi [ ir<%rdx>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[VP15]]> from scalar.ph)
; CHECK-NEXT: IR %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ] (extra operand: vp<[[VP16]]> from scalar.ph)
; CHECK-NEXT: IR %gep = getelementptr i32, ptr %p, i32 %iv
; CHECK-NEXT: IR %x = load i32, ptr %gep, align 4
; CHECK-NEXT: IR %rdx.next = add i32 %rdx, %x
; CHECK-NEXT: IR %iv.next = add i32 %iv, 1
; CHECK-NEXT: IR %ec = icmp eq i32 %iv.next, %n
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop
loop:
%iv = phi i32 [0, %entry], [%iv.next, %loop]
%rdx = phi i32 [0, %entry], [%rdx.next, %loop]
%gep = getelementptr i32, ptr %p, i32 %iv
%x = load i32, ptr %gep
%rdx.next = add i32 %rdx, %x
%iv.next = add i32 %iv, 1
%ec = icmp eq i32 %iv.next, %n
br i1 %ec, label %exit, label %loop
exit:
ret i32 %rdx.next
}
Reference in New Issue View Git Blame Copy Permalink