ede1a9626b
Building on top of the recent changes to introduce BranchOnTwoConds, this patch adds support for vectorizing loops with multiple early exits, all dominating a countable latch. The early exits must form a dominance chain, so we can simply check which early exit has been taken in dominance order. Currently LoopVectorizationLegality ensures that all exits other than the latch must be uncountable. handleUncountableEarlyExits now collects those uncountable exits and processes each exit. In the vector region, we compute if any exit has been taken, by taking the OR of all early exit conditions (EarlyExitConds) and checking if there's any active lane. If the early exit is taken, we exit the loop and compute which early exit has been taken. The first taken early exit is the one where its exit condition is true in the first active lane of EarlyExitConds. We create a chain of dispatch blocks outside the loop to check this for the early exit blocks ordered by dominance. Depends on https://github.com/llvm/llvm-project/pull/174016. PR: https://github.com/llvm/llvm-project/pull/174864
193 lines
8.0 KiB
LLVM
193 lines
8.0 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
|
|
; RUN: opt -S < %s -p loop-vectorize | FileCheck %s
|
|
|
|
declare void @init_mem(ptr, i64);
|
|
|
|
define i64 @one_uncountable_two_countable_same_exit_phi_of_consts() {
|
|
; CHECK-LABEL: define i64 @one_uncountable_two_countable_same_exit_phi_of_consts() {
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
|
|
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
|
|
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
|
|
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
|
|
; CHECK-NEXT: br label [[LOOP:%.*]]
|
|
; CHECK: loop:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
|
|
; CHECK-NEXT: [[CMP1:%.*]] = icmp ne i64 [[INDEX]], 64
|
|
; CHECK-NEXT: br i1 [[CMP1]], label [[SEARCH:%.*]], label [[LOOP_END:%.*]]
|
|
; CHECK: search:
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
|
|
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
|
|
; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
|
|
; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_END]], label [[LOOP_INC]]
|
|
; CHECK: loop.inc:
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
|
|
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 128
|
|
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
|
|
; CHECK: loop.end:
|
|
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 0, [[LOOP]] ], [ 1, [[SEARCH]] ], [ 0, [[LOOP_INC]] ]
|
|
; CHECK-NEXT: ret i64 [[RETVAL]]
|
|
;
|
|
entry:
|
|
%p1 = alloca [1024 x i8]
|
|
%p2 = alloca [1024 x i8]
|
|
call void @init_mem(ptr %p1, i64 1024)
|
|
call void @init_mem(ptr %p2, i64 1024)
|
|
br label %loop
|
|
|
|
loop:
|
|
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
|
|
%cmp1 = icmp ne i64 %index, 64
|
|
br i1 %cmp1, label %search, label %loop.end
|
|
|
|
search:
|
|
%arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
|
|
%ld1 = load i8, ptr %arrayidx, align 1
|
|
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
|
|
%ld2 = load i8, ptr %arrayidx1, align 1
|
|
%cmp3 = icmp eq i8 %ld1, %ld2
|
|
br i1 %cmp3, label %loop.end, label %loop.inc
|
|
|
|
loop.inc:
|
|
%index.next = add i64 %index, 1
|
|
%exitcond = icmp ne i64 %index.next, 128
|
|
br i1 %exitcond, label %loop, label %loop.end
|
|
|
|
loop.end:
|
|
%retval = phi i64 [ 0, %loop ], [ 1, %search ], [ 0, %loop.inc ]
|
|
ret i64 %retval
|
|
}
|
|
|
|
|
|
define i64 @one_uncountable_two_countable_diff_exit_no_phis() {
|
|
; CHECK-LABEL: define i64 @one_uncountable_two_countable_diff_exit_no_phis() {
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
|
|
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
|
|
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
|
|
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
|
|
; CHECK-NEXT: br label [[LOOP:%.*]]
|
|
; CHECK: loop:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
|
|
; CHECK-NEXT: [[CMP1:%.*]] = icmp ne i64 [[INDEX]], 64
|
|
; CHECK-NEXT: br i1 [[CMP1]], label [[SEARCH:%.*]], label [[LOOP_END:%.*]]
|
|
; CHECK: search:
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
|
|
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
|
|
; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
|
|
; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_END_EARLY:%.*]], label [[LOOP_INC]]
|
|
; CHECK: loop.inc:
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
|
|
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 128
|
|
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
|
|
; CHECK: loop.end.early:
|
|
; CHECK-NEXT: ret i64 1
|
|
; CHECK: loop.end:
|
|
; CHECK-NEXT: ret i64 0
|
|
;
|
|
entry:
|
|
%p1 = alloca [1024 x i8]
|
|
%p2 = alloca [1024 x i8]
|
|
call void @init_mem(ptr %p1, i64 1024)
|
|
call void @init_mem(ptr %p2, i64 1024)
|
|
br label %loop
|
|
|
|
loop:
|
|
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
|
|
%cmp1 = icmp ne i64 %index, 64
|
|
br i1 %cmp1, label %search, label %loop.end
|
|
|
|
search:
|
|
%arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
|
|
%ld1 = load i8, ptr %arrayidx, align 1
|
|
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
|
|
%ld2 = load i8, ptr %arrayidx1, align 1
|
|
%cmp3 = icmp eq i8 %ld1, %ld2
|
|
br i1 %cmp3, label %loop.end.early, label %loop.inc
|
|
|
|
loop.inc:
|
|
%index.next = add i64 %index, 1
|
|
%exitcond = icmp ne i64 %index.next, 128
|
|
br i1 %exitcond, label %loop, label %loop.end
|
|
|
|
loop.end.early:
|
|
ret i64 1
|
|
|
|
loop.end:
|
|
ret i64 0
|
|
}
|
|
|
|
; Test that early exit with a live-in condition (function argument) combined
|
|
; with a dereferenceable load-based condition does not crash. The live-in
|
|
; condition has no defining recipe in VPlan.
|
|
define i64 @early_exit_with_live_in_condition(i1 %cond) {
|
|
; CHECK-LABEL: define i64 @early_exit_with_live_in_condition(
|
|
; CHECK-SAME: i1 [[COND:%.*]]) {
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[P:%.*]] = alloca [1024 x i8], align 1
|
|
; CHECK-NEXT: call void @init_mem(ptr [[P]], i64 1024)
|
|
; CHECK-NEXT: br label [[LOOP:%.*]]
|
|
; CHECK: vector.ph:
|
|
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[COND]], i64 0
|
|
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK: vector.body:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[LOOP]] ], [ [[INC:%.*]], [[VECTOR_BODY_INTERIM:%.*]] ]
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IV]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[ARRAYIDX]], align 1
|
|
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
|
|
; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> splat (i1 true), <4 x i1> [[TMP1]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
|
|
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], 1024
|
|
; CHECK-NEXT: br i1 [[TMP4]], label [[VECTOR_EARLY_EXIT_CHECK:%.*]], label [[VECTOR_BODY_INTERIM]]
|
|
; CHECK: vector.body.interim:
|
|
; CHECK-NEXT: br i1 [[EXITCOND]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; CHECK: middle.block:
|
|
; CHECK-NEXT: br label [[LOOP_END:%.*]]
|
|
; CHECK: vector.early.exit.check:
|
|
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i32 0
|
|
; CHECK-NEXT: br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT_0:%.*]], label [[VECTOR_EARLY_EXIT_1:%.*]]
|
|
; CHECK: vector.early.exit.1:
|
|
; CHECK-NEXT: br label [[LOOP_END]]
|
|
; CHECK: vector.early.exit.0:
|
|
; CHECK-NEXT: br label [[LOOP_END]]
|
|
; CHECK: loop.end:
|
|
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 1, [[VECTOR_EARLY_EXIT_1]] ], [ 0, [[VECTOR_EARLY_EXIT_0]] ], [ 2, [[MIDDLE_BLOCK]] ]
|
|
; CHECK-NEXT: ret i64 [[RETVAL]]
|
|
;
|
|
entry:
|
|
%p = alloca [1024 x i8]
|
|
call void @init_mem(ptr %p, i64 1024)
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i64 [ %inc, %latch ], [ 0, %entry ]
|
|
br i1 %cond, label %loop.end, label %search
|
|
|
|
search:
|
|
%arrayidx = getelementptr inbounds i8, ptr %p, i64 %iv
|
|
%ld = load i8, ptr %arrayidx, align 1
|
|
%cmp = icmp eq i8 %ld, 0
|
|
br i1 %cmp, label %loop.end, label %latch
|
|
|
|
latch:
|
|
%inc = add nuw nsw i64 %iv, 1
|
|
%exitcond = icmp eq i64 %inc, 1024
|
|
br i1 %exitcond, label %loop.end, label %loop
|
|
|
|
loop.end:
|
|
%retval = phi i64 [ 0, %loop ], [ 1, %search ], [ 2, %latch ]
|
|
ret i64 %retval
|
|
}
|
|
;.
|
|
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
|
|
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
|
|
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
|
|
;.
|