f492eb9509
AnyOf reduces multiple input vectors to a single boolean value. When used for early-exit vectorization, we need to consider any lane after the early exit being poison. Any poison lane would result in poison after the AnyOf reduction. To prevent this, freeze all inputs to AnyOf. Fixes https://github.com/llvm/llvm-project/issues/153946. Fixes https://github.com/llvm/llvm-project/issues/155162. https://alive2.llvm.org/ce/z/FD-XxA PR: https://github.com/llvm/llvm-project/pull/154156
99 lines
4.7 KiB
LLVM
99 lines
4.7 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
|
|
; RUN: opt -p loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 -S %s | FileCheck --check-prefix=VF4IC4 %s
|
|
|
|
declare void @init_mem(ptr, i64);
|
|
|
|
define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
|
|
; VF4IC4-LABEL: define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
|
|
; VF4IC4-NEXT: [[ENTRY:.*:]]
|
|
; VF4IC4-NEXT: [[SRC:%.*]] = alloca [128 x i32], align 4
|
|
; VF4IC4-NEXT: call void @init_mem(ptr [[SRC]])
|
|
; VF4IC4-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; VF4IC4: [[VECTOR_PH]]:
|
|
; VF4IC4-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; VF4IC4: [[VECTOR_BODY]]:
|
|
; VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; VF4IC4-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
|
|
; VF4IC4-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 4
|
|
; VF4IC4-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 8
|
|
; VF4IC4-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 12
|
|
; VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; VF4IC4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; VF4IC4-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
|
|
; VF4IC4-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4
|
|
; VF4IC4-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 10)
|
|
; VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], splat (i32 10)
|
|
; VF4IC4-NEXT: [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD2]], splat (i32 10)
|
|
; VF4IC4-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD3]], splat (i32 10)
|
|
; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
|
|
; VF4IC4-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP2]]
|
|
; VF4IC4-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP6]]
|
|
; VF4IC4-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP14]], [[TMP9]]
|
|
; VF4IC4-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP7]]
|
|
; VF4IC4-NEXT: [[TMP16:%.*]] = or <4 x i1> [[TMP10]], [[TMP15]]
|
|
; VF4IC4-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP8]]
|
|
; VF4IC4-NEXT: [[TMP11:%.*]] = or <4 x i1> [[TMP16]], [[TMP17]]
|
|
; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
|
|
; VF4IC4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
|
|
; VF4IC4-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
|
|
; VF4IC4-NEXT: br i1 [[TMP5]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; VF4IC4: [[MIDDLE_SPLIT]]:
|
|
; VF4IC4-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
|
|
; VF4IC4: [[MIDDLE_BLOCK]]:
|
|
; VF4IC4-NEXT: br label %[[E2:.*]]
|
|
; VF4IC4: [[VECTOR_EARLY_EXIT]]:
|
|
; VF4IC4-NEXT: br label %[[E1:.*]]
|
|
; VF4IC4: [[SCALAR_PH]]:
|
|
; VF4IC4-NEXT: br label %[[LOOP_HEADER:.*]]
|
|
; VF4IC4: [[LOOP_HEADER]]:
|
|
; VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[SCALAR_PH]] ]
|
|
; VF4IC4-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
|
|
; VF4IC4-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
|
|
; VF4IC4-NEXT: [[C_1:%.*]] = icmp eq i32 [[L]], 10
|
|
; VF4IC4-NEXT: br i1 [[C_1]], label %[[E1]], label %[[LOOP_LATCH]]
|
|
; VF4IC4: [[LOOP_LATCH]]:
|
|
; VF4IC4-NEXT: [[INC]] = add nuw i64 [[IV]], 1
|
|
; VF4IC4-NEXT: [[C_2:%.*]] = icmp eq i64 [[INC]], 128
|
|
; VF4IC4-NEXT: br i1 [[C_2]], label %[[E2]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
|
|
; VF4IC4: [[E1]]:
|
|
; VF4IC4-NEXT: [[P1:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ]
|
|
; VF4IC4-NEXT: ret i64 [[P1]]
|
|
; VF4IC4: [[E2]]:
|
|
; VF4IC4-NEXT: [[P2:%.*]] = phi i64 [ 1, %[[LOOP_LATCH]] ], [ 1, %[[MIDDLE_BLOCK]] ]
|
|
; VF4IC4-NEXT: ret i64 [[P2]]
|
|
;
|
|
entry:
|
|
%src = alloca [128 x i32]
|
|
call void @init_mem(ptr %src)
|
|
br label %loop.header
|
|
|
|
loop.header:
|
|
%iv = phi i64 [ %inc, %loop.latch ], [ 0, %entry ]
|
|
%gep.src = getelementptr inbounds i32, ptr %src, i64 %iv
|
|
%l = load i32, ptr %gep.src
|
|
%c.1 = icmp eq i32 %l, 10
|
|
br i1 %c.1, label %e1, label %loop.latch
|
|
|
|
loop.latch:
|
|
%inc = add nuw i64 %iv, 1
|
|
%c.2 = icmp eq i64 %inc, 128
|
|
br i1 %c.2, label %e2, label %loop.header, !llvm.loop !0
|
|
|
|
e1:
|
|
%p1 = phi i64 [ 0, %loop.header ]
|
|
ret i64 %p1
|
|
|
|
e2:
|
|
%p2 = phi i64 [ 1, %loop.latch ]
|
|
ret i64 %p2
|
|
}
|
|
|
|
!0 = distinct !{!0, !1}
|
|
!1 = !{!"llvm.loop.interleave.count", i32 4}
|
|
;.
|
|
; VF4IC4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
|
|
; VF4IC4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
|
|
; VF4IC4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
|
|
; VF4IC4: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
|
|
;.
|