d43213fe80
This reverts commit b0b3e3e1c7f6387eabc2ef9ff1fea311e63a4299. After thinking about this for a bit, I don't think this is correct. vscale being a power-of-2 only guarantees the canonical IV increment overflows to zero, but not overflows in general.
100 lines
6.2 KiB
LLVM
100 lines
6.2 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
|
|
; RUN: opt -p loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-vector-width=4 -S %s | FileCheck %s
|
|
|
|
define i32 @find_last_trunc_iv(ptr %src, i64 %n) {
|
|
; CHECK-LABEL: define i32 @find_last_trunc_iv(
|
|
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
|
|
; CHECK-NEXT: [[LOOP:.*:]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
|
|
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
|
|
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
|
|
; CHECK: [[VECTOR_BODY1]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE7:.*]] ]
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE7]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP32:%.*]], %[[PRED_LOAD_CONTINUE7]] ]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP31:%.*]], %[[PRED_LOAD_CONTINUE7]] ]
|
|
; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT8:%.*]], %[[PRED_LOAD_CONTINUE7]] ]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
|
|
; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
|
|
; CHECK: [[PRED_LOAD_IF]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = add i64 [[INDEX]], 0
|
|
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
|
|
; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
|
|
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[L]], i32 0
|
|
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
|
|
; CHECK: [[PRED_LOAD_CONTINUE]]:
|
|
; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY1]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
|
|
; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
|
|
; CHECK: [[PRED_LOAD_IF2]]:
|
|
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 1
|
|
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP10]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
|
|
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i32 1
|
|
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE3]]
|
|
; CHECK: [[PRED_LOAD_CONTINUE3]]:
|
|
; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF2]] ]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
|
|
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT1:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]
|
|
; CHECK: [[EXIT1]]:
|
|
; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], 2
|
|
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP16]]
|
|
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4
|
|
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i32 2
|
|
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]]
|
|
; CHECK: [[PRED_LOAD_CONTINUE5]]:
|
|
; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE3]] ], [ [[TMP19]], %[[EXIT1]] ]
|
|
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
|
|
; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF6:.*]], label %[[PRED_LOAD_CONTINUE7]]
|
|
; CHECK: [[PRED_LOAD_IF6]]:
|
|
; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], 3
|
|
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP22]]
|
|
; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
|
|
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i32 3
|
|
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
|
|
; CHECK: [[PRED_LOAD_CONTINUE7]]:
|
|
; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], %[[PRED_LOAD_IF6]] ]
|
|
; CHECK-NEXT: [[TMP27:%.*]] = icmp eq <4 x i32> [[TMP26]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP28:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP27]], <4 x i1> zeroinitializer
|
|
; CHECK-NEXT: [[TMP29:%.*]] = freeze <4 x i1> [[TMP28]]
|
|
; CHECK-NEXT: [[TMP30:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP29]])
|
|
; CHECK-NEXT: [[TMP31]] = select i1 [[TMP30]], <4 x i1> [[TMP28]], <4 x i1> [[TMP1]]
|
|
; CHECK-NEXT: [[TMP32]] = select i1 [[TMP30]], <4 x i32> [[VEC_IND1]], <4 x i32> [[VEC_PHI]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
|
|
; CHECK-NEXT: [[VEC_IND_NEXT8]] = add <4 x i32> [[VEC_IND1]], splat (i32 4)
|
|
; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[RDX_NEXT_LCSSA:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP32]], <4 x i1> [[TMP31]], i32 0)
|
|
; CHECK-NEXT: br label %[[EXIT:.*]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: ret i32 [[RDX_NEXT_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
|
|
%rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]
|
|
%gep.src = getelementptr inbounds i32, ptr %src, i64 %iv
|
|
%l = load i32, ptr %gep.src
|
|
%cmp103 = icmp eq i32 %l, 0
|
|
%0 = trunc i64 %iv to i32
|
|
%rdx.next = select i1 %cmp103, i32 %0, i32 %rdx
|
|
%iv.next = add i64 %iv, 1
|
|
%ec = icmp eq i64 %iv, %n
|
|
br i1 %ec, label %exit, label %loop
|
|
|
|
exit:
|
|
ret i32 %rdx.next
|
|
}
|