; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
; RUN: opt -S < %s -p loop-vectorize | FileCheck %s

declare void @init_mem(ptr, i64);

define i64 @one_uncountable_two_countable_same_exit_phi_of_consts() {
; CHECK-LABEL: define i64 @one_uncountable_two_countable_same_exit_phi_of_consts() {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT:    br label [[LOOP:%.*]]
; CHECK:       loop:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i64 [[INDEX]], 64
; CHECK-NEXT:    br i1 [[CMP1]], label [[SEARCH:%.*]], label [[LOOP_END:%.*]]
; CHECK:       search:
; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_END]], label [[LOOP_INC]]
; CHECK:       loop.inc:
; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 128
; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
; CHECK:       loop.end:
; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 0, [[LOOP]] ], [ 1, [[SEARCH]] ], [ 0, [[LOOP_INC]] ]
; CHECK-NEXT:    ret i64 [[RETVAL]]
;
entry:
  %p1 = alloca [1024 x i8]
  %p2 = alloca [1024 x i8]
  call void @init_mem(ptr %p1, i64 1024)
  call void @init_mem(ptr %p2, i64 1024)
  br label %loop

loop:
  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
  %cmp1 = icmp ne i64 %index, 64
  br i1 %cmp1, label %search, label %loop.end

search:
  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
  %ld1 = load i8, ptr %arrayidx, align 1
  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
  %ld2 = load i8, ptr %arrayidx1, align 1
  %cmp3 = icmp eq i8 %ld1, %ld2
  br i1 %cmp3, label %loop.end, label %loop.inc

loop.inc:
  %index.next = add i64 %index, 1
  %exitcond = icmp ne i64 %index.next, 128
  br i1 %exitcond, label %loop, label %loop.end

loop.end:
  %retval = phi i64 [ 0, %loop ], [ 1, %search ], [ 0, %loop.inc ]
  ret i64 %retval
}


define i64 @one_uncountable_two_countable_diff_exit_no_phis() {
; CHECK-LABEL: define i64 @one_uncountable_two_countable_diff_exit_no_phis() {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT:    br label [[LOOP:%.*]]
; CHECK:       loop:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i64 [[INDEX]], 64
; CHECK-NEXT:    br i1 [[CMP1]], label [[SEARCH:%.*]], label [[LOOP_END:%.*]]
; CHECK:       search:
; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_END_EARLY:%.*]], label [[LOOP_INC]]
; CHECK:       loop.inc:
; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 128
; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
; CHECK:       loop.end.early:
; CHECK-NEXT:    ret i64 1
; CHECK:       loop.end:
; CHECK-NEXT:    ret i64 0
;
entry:
  %p1 = alloca [1024 x i8]
  %p2 = alloca [1024 x i8]
  call void @init_mem(ptr %p1, i64 1024)
  call void @init_mem(ptr %p2, i64 1024)
  br label %loop

loop:
  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
  %cmp1 = icmp ne i64 %index, 64
  br i1 %cmp1, label %search, label %loop.end

search:
  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
  %ld1 = load i8, ptr %arrayidx, align 1
  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
  %ld2 = load i8, ptr %arrayidx1, align 1
  %cmp3 = icmp eq i8 %ld1, %ld2
  br i1 %cmp3, label %loop.end.early, label %loop.inc

loop.inc:
  %index.next = add i64 %index, 1
  %exitcond = icmp ne i64 %index.next, 128
  br i1 %exitcond, label %loop, label %loop.end

loop.end.early:
  ret i64 1

loop.end:
  ret i64 0
}

; Test that early exit with a live-in condition (function argument) combined
; with a dereferenceable load-based condition does not crash. The live-in
; condition has no defining recipe in VPlan.
define i64 @early_exit_with_live_in_condition(i1 %cond) {
; CHECK-LABEL: define i64 @early_exit_with_live_in_condition(
; CHECK-SAME: i1 [[COND:%.*]]) {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[P:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT:    call void @init_mem(ptr [[P]], i64 1024)
; CHECK-NEXT:    br label [[LOOP:%.*]]
; CHECK:       vector.ph:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[COND]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
; CHECK:       vector.body:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[LOOP]] ], [ [[INC:%.*]], [[VECTOR_BODY_INTERIM:%.*]] ]
; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IV]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT:    [[INC]] = add nuw i64 [[IV]], 4
; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> splat (i1 true), <4 x i1> [[TMP1]]
; CHECK-NEXT:    [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
; CHECK-NEXT:    [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], 1024
; CHECK-NEXT:    br i1 [[TMP4]], label [[VECTOR_EARLY_EXIT_CHECK:%.*]], label [[VECTOR_BODY_INTERIM]]
; CHECK:       vector.body.interim:
; CHECK-NEXT:    br i1 [[EXITCOND]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       middle.block:
; CHECK-NEXT:    br label [[LOOP_END:%.*]]
; CHECK:       vector.early.exit.check:
; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i32 0
; CHECK-NEXT:    br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT_0:%.*]], label [[VECTOR_EARLY_EXIT_1:%.*]]
; CHECK:       vector.early.exit.1:
; CHECK-NEXT:    br label [[LOOP_END]]
; CHECK:       vector.early.exit.0:
; CHECK-NEXT:    br label [[LOOP_END]]
; CHECK:       loop.end:
; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 1, [[VECTOR_EARLY_EXIT_1]] ], [ 0, [[VECTOR_EARLY_EXIT_0]] ], [ 2, [[MIDDLE_BLOCK]] ]
; CHECK-NEXT:    ret i64 [[RETVAL]]
;
entry:
  %p = alloca [1024 x i8]
  call void @init_mem(ptr %p, i64 1024)
  br label %loop

loop:
  %iv = phi i64 [ %inc, %latch ], [ 0, %entry ]
  br i1 %cond, label %loop.end, label %search

search:
  %arrayidx = getelementptr inbounds i8, ptr %p, i64 %iv
  %ld = load i8, ptr %arrayidx, align 1
  %cmp = icmp eq i8 %ld, 0
  br i1 %cmp, label %loop.end, label %latch

latch:
  %inc = add nuw nsw i64 %iv, 1
  %exitcond = icmp eq i64 %inc, 1024
  br i1 %exitcond, label %loop.end, label %loop

loop.end:
  %retval = phi i64 [ 0, %loop ], [ 1, %search ], [ 2, %latch ]
  ret i64 %retval
}
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
;.