90c1c588f8
The wrap flags from the IV bin-op are not guaranteed to apply to truncated inductions, which are evaluated in narrower types. Instead of dropping them late (in expandVPWidenIntOrFpInduction), do not add them at the outset, the prevent invalid transforms based on incorrect flags in the future. PR: https://github.com/llvm/llvm-project/pull/188966
211 lines
8.3 KiB
LLVM
211 lines
8.3 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
|
|
; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -disable-output -vplan-print-after="optimize$" %s 2>&1 | FileCheck %s
|
|
|
|
define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
|
|
; CHECK-LABEL: VPlan for loop in 'iv_no_binary_op_in_descriptor'
|
|
; CHECK: VPlan 'Initial VPlan for VF={8},UF>=1' {
|
|
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
|
|
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
|
|
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
|
|
; CHECK-NEXT: Live-in ir<1000> = original trip-count
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<entry>:
|
|
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: vector.ph:
|
|
; CHECK-NEXT: Successor(s): vector loop
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: <x1> vector loop: {
|
|
; CHECK-NEXT: vector.body:
|
|
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
|
|
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
|
|
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
|
|
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
|
|
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
|
|
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%iv>
|
|
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
|
|
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-NEXT: }
|
|
; CHECK-NEXT: Successor(s): middle.block
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: middle.block:
|
|
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<1000>, vp<[[VP2]]>
|
|
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
|
|
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<exit>:
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: scalar.ph:
|
|
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
|
|
; CHECK-NEXT: Successor(s): ir-bb<loop.header>
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<loop.header>:
|
|
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next.p, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
|
|
; CHECK-NEXT: IR %gep = getelementptr inbounds i64, ptr %dst, i64 %iv
|
|
; CHECK-NEXT: IR store i64 %iv, ptr %gep, align 8
|
|
; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-NEXT: }
|
|
;
|
|
entry:
|
|
br label %loop.header
|
|
|
|
loop.header:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next.p, %loop.latch ]
|
|
%gep = getelementptr inbounds i64, ptr %dst, i64 %iv
|
|
store i64 %iv, ptr %gep, align 8
|
|
%iv.next = add i64 %iv, 1
|
|
br label %loop.latch
|
|
|
|
loop.latch:
|
|
%iv.next.p = phi i64 [ %iv.next, %loop.header ]
|
|
%exitcond.not = icmp eq i64 %iv.next.p, 1000
|
|
br i1 %exitcond.not, label %exit, label %loop.header
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
; Check that VPWidenIntOrFPInductionRecipe is expanded into smaller recipes in
|
|
; the final VPlan.
|
|
define void @iv_expand(ptr %p, i64 %n) {
|
|
; CHECK-LABEL: VPlan for loop in 'iv_expand'
|
|
; CHECK: VPlan 'Initial VPlan for VF={8},UF>=1' {
|
|
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
|
|
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
|
|
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
|
|
; CHECK-NEXT: Live-in ir<%n> = original trip-count
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<entry>:
|
|
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: vector.ph:
|
|
; CHECK-NEXT: Successor(s): vector loop
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: <x1> vector loop: {
|
|
; CHECK-NEXT: vector.body:
|
|
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
|
|
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
|
|
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
|
|
; CHECK-NEXT: CLONE ir<%q> = getelementptr ir<%p>, vp<[[VP4]]>
|
|
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%q>
|
|
; CHECK-NEXT: WIDEN ir<%x> = load vp<[[VP5]]>
|
|
; CHECK-NEXT: WIDEN ir<%y> = add ir<%x>, ir<%iv>
|
|
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%q>
|
|
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%y>
|
|
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
|
|
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-NEXT: }
|
|
; CHECK-NEXT: Successor(s): middle.block
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: middle.block:
|
|
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
|
|
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
|
|
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<exit>:
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: scalar.ph:
|
|
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
|
|
; CHECK-NEXT: Successor(s): ir-bb<loop>
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<loop>:
|
|
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
|
|
; CHECK-NEXT: IR %q = getelementptr i64, ptr %p, i64 %iv
|
|
; CHECK-NEXT: IR %x = load i64, ptr %q, align 4
|
|
; CHECK-NEXT: IR %y = add i64 %x, %iv
|
|
; CHECK-NEXT: IR store i64 %y, ptr %q, align 4
|
|
; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
|
|
; CHECK-NEXT: IR %done = icmp eq i64 %iv.next, %n
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-NEXT: }
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i64 [0, %entry], [%iv.next, %loop]
|
|
%q = getelementptr i64, ptr %p, i64 %iv
|
|
%x = load i64, ptr %q
|
|
%y = add i64 %x, %iv
|
|
store i64 %y, ptr %q
|
|
%iv.next = add i64 %iv, 1
|
|
%done = icmp eq i64 %iv.next, %n
|
|
br i1 %done, label %exit, label %loop
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
; FIXME: The nuw/nsw flags from the scalar IV are not valid for the truncated
|
|
; IV.
|
|
define void @iv_truncated_wrap_flags(ptr %dst, i64 %n) {
|
|
; CHECK-LABEL: VPlan for loop in 'iv_truncated_wrap_flags'
|
|
; CHECK: VPlan 'Initial VPlan for VF={8},UF>=1' {
|
|
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
|
|
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
|
|
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
|
|
; CHECK-NEXT: Live-in ir<%n> = original trip-count
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<entry>:
|
|
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: vector.ph:
|
|
; CHECK-NEXT: Successor(s): vector loop
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: <x1> vector loop: {
|
|
; CHECK-NEXT: vector.body:
|
|
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
|
|
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i16)
|
|
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
|
|
; CHECK-NEXT: CLONE ir<%ptr> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
|
|
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%ptr>
|
|
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%iv>
|
|
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
|
|
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-NEXT: }
|
|
; CHECK-NEXT: Successor(s): middle.block
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: middle.block:
|
|
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
|
|
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
|
|
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<exit>:
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: scalar.ph:
|
|
; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
|
|
; CHECK-NEXT: Successor(s): ir-bb<loop>
|
|
; CHECK-EMPTY:
|
|
; CHECK-NEXT: ir-bb<loop>:
|
|
; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
|
|
; CHECK-NEXT: IR %val = trunc i64 %iv to i16
|
|
; CHECK-NEXT: IR %ptr = getelementptr inbounds i16, ptr %dst, i64 %iv
|
|
; CHECK-NEXT: IR store i16 %val, ptr %ptr, align 2
|
|
; CHECK-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
|
|
; CHECK-NEXT: IR %exit.cond = icmp eq i64 %iv.next, %n
|
|
; CHECK-NEXT: No successors
|
|
; CHECK-NEXT: }
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
|
|
%val = trunc i64 %iv to i16
|
|
%ptr = getelementptr inbounds i16, ptr %dst, i64 %iv
|
|
store i16 %val, ptr %ptr, align 2
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%exit.cond = icmp eq i64 %iv.next, %n
|
|
br i1 %exit.cond, label %exit, label %loop
|
|
|
|
exit:
|
|
ret void
|
|
}
|