86813aa786
Epilogue vectorization currently relies on the resume phi for the canonical induction being always available, which is why VPPhi are considered to have side-effects, to prevent their removal. This patch adds a new ResumeForEpilogue opcode to mark the resume phi as used for epilogue vectorization. This allows treating VPPhis in general as not having side-effects, enabling removal of unused VPPhis.
63 lines
3.2 KiB
LLVM
63 lines
3.2 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
|
|
; RUN: opt -passes=loop-vectorize -force-tail-folding-style=data -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-target-supports-scalable-vectors -S < %s | FileCheck %s
|
|
|
|
; vscale is not guaranteed to be a power of two, so this test (which
|
|
; deliberately doesn't correspond to an in-tree backend since those
|
|
; *do* have vscale as power-of-two) exercises the code required for the
|
|
; minimum iteration check in the non-power-of-two case.
|
|
define void @foo(i32 %val, ptr dereferenceable(1024) %ptr) {
|
|
; CHECK-LABEL: @foo(
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
|
|
; CHECK-NEXT: [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i64 -257, [[TMP7]]
|
|
; CHECK-NEXT: br i1 [[TMP8]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
|
|
; CHECK: vector.ph:
|
|
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
|
|
; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], 1
|
|
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 256, [[TMP2]]
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK: vector.body:
|
|
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 256)
|
|
; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; CHECK: middle.block:
|
|
; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT:%.*]]
|
|
; CHECK: scalar.ph:
|
|
; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
|
|
; CHECK: while.body:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
|
|
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP]], align 4
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
|
|
; CHECK-NEXT: [[CMP10:%.*]] = icmp ult i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[CMP10]], label [[WHILE_BODY]], label [[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
|
|
; CHECK: while.end.loopexit:
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
entry:
|
|
br label %while.body
|
|
|
|
while.body: ; preds = %while.body, %entry
|
|
%index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]
|
|
%gep = getelementptr i32, ptr %ptr, i64 %index
|
|
%ld1 = load i32, ptr %gep, align 4
|
|
%index.next = add nsw i64 %index, 1
|
|
%cmp10 = icmp ult i64 %index.next, 256
|
|
br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0
|
|
|
|
while.end.loopexit: ; preds = %while.body
|
|
ret void
|
|
}
|
|
|
|
!0 = distinct !{!0, !1, !2, !3, !4}
|
|
!1 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
|
|
!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
|
|
!3 = !{!"llvm.loop.interleave.count", i32 1}
|
|
!4 = !{!"llvm.loop.vectorize.width", i32 4}
|