8c6658aca6
When a recipe can be safely sunk and all of its users are outside the vector loop region in the same dedicated exit block, the recipe does not need to be executed on every iteration. This patch extends the VPlan-based LICM (Loop Invariant Code Motion) to also sink such recipes from the vector loop region into the exit block. This reduces redundant computation and improves cost model accuracy. TODO: Support nested loop sinking TODO: Support sinking `VPReplicateRecipe` (requires `replicateByVF` fixes) TODO: Support recipes with multiple defined values (e.g., interleaved loads) TODO: Clone recipes without users to all exit blocks TODO: Support PHI node users by checking incoming value blocks TODO: Support sinking when users are in multiple blocks TODO: Clone recipes when users are on multiple exit paths Co-authored-by: Luke Lau <luke@igalia.com> --------- Co-authored-by: Luke Lau <luke@igalia.com> Co-authored-by: Luke Lau <luke_lau@icloud.com>
194 lines
11 KiB
LLVM
194 lines
11 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
|
|
; RUN: opt -p loop-vectorize -mtriple=x86_64-apple-macosx -mcpu=penryn -S %s | FileCheck %s
|
|
|
|
define i64 @test_value_in_exit_compare_chain_used_outside(ptr %src, i64 %x, i64 range(i64 1, 32) %N) {
|
|
; CHECK-LABEL: define i64 @test_value_in_exit_compare_chain_used_outside(
|
|
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[X:%.*]], i64 range(i64 1, 32) [[N:%.*]]) #[[ATTR0:[0-9]+]] {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -1
|
|
; CHECK-NEXT: [[TMP1:%.*]] = freeze i64 [[TMP0]]
|
|
; CHECK-NEXT: [[UMIN2:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[X]])
|
|
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[UMIN2]], 1
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 8
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
|
|
; CHECK: [[VECTOR_SCEVCHECK]]:
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[N]], -1
|
|
; CHECK-NEXT: [[TMP4:%.*]] = freeze i64 [[TMP3]]
|
|
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP4]], i64 [[X]])
|
|
; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[UMIN]] to i1
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i64 [[UMIN]], 1
|
|
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
|
|
; CHECK-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
|
|
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 8, i64 [[N_MOD_VF]]
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP9]]
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i8> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i8> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP18:%.*]] = and i64 [[TMP10]], 1
|
|
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP18]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP26]], i64 0
|
|
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 -3
|
|
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP26]], i64 -4
|
|
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[TMP14]], i64 -3
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
|
|
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP15]], align 1
|
|
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
|
|
; CHECK-NEXT: [[REVERSE5:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD4]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
|
|
; CHECK-NEXT: [[TMP16]] = xor <4 x i8> [[REVERSE]], [[VEC_PHI]]
|
|
; CHECK-NEXT: [[TMP17]] = xor <4 x i8> [[REVERSE5]], [[VEC_PHI3]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], 8
|
|
; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[BIN_RDX:%.*]] = xor <4 x i8> [[TMP17]], [[TMP16]]
|
|
; CHECK-NEXT: [[TMP19:%.*]] = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> [[BIN_RDX]])
|
|
; CHECK-NEXT: br label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
|
|
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
|
|
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
|
|
; CHECK: [[LOOP_HEADER]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
|
|
; CHECK-NEXT: [[XOR_RED:%.*]] = phi i8 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[XOR_RED_NEXT:%.*]], %[[LOOP_LATCH]] ]
|
|
; CHECK-NEXT: [[IV_AND:%.*]] = and i64 [[IV]], 1
|
|
; CHECK-NEXT: [[X_INC:%.*]] = add i64 [[IV_AND]], [[X]]
|
|
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV_AND]]
|
|
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[X_INC]], 0
|
|
; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT_1:.*]], label %[[LOOP_LATCH]]
|
|
; CHECK: [[LOOP_LATCH]]:
|
|
; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1
|
|
; CHECK-NEXT: [[XOR_RED_NEXT]] = xor i8 [[L]], [[XOR_RED]]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
|
|
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_2:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
|
|
; CHECK: [[EXIT_1]]:
|
|
; CHECK-NEXT: [[X_INC_LCSSA:%.*]] = phi i64 [ [[X_INC]], %[[LOOP_HEADER]] ]
|
|
; CHECK-NEXT: ret i64 [[X_INC_LCSSA]]
|
|
; CHECK: [[EXIT_2]]:
|
|
; CHECK-NEXT: [[XOR_RED_NEXT_LCSSA:%.*]] = phi i8 [ [[XOR_RED_NEXT]], %[[LOOP_LATCH]] ]
|
|
; CHECK-NEXT: [[R:%.*]] = zext i8 [[XOR_RED_NEXT_LCSSA]] to i64
|
|
; CHECK-NEXT: ret i64 [[R]]
|
|
;
|
|
entry:
|
|
br label %loop.header
|
|
|
|
loop.header:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
|
|
%xor.red = phi i8 [ 0, %entry ], [ %xor.red.next, %loop.latch ]
|
|
%iv.and = and i64 %iv, 1
|
|
%x.inc = add i64 %iv.and, %x
|
|
%gep.src = getelementptr i8, ptr %src, i64 %iv.and
|
|
%cmp = icmp eq i64 %x.inc, 0
|
|
br i1 %cmp, label %exit.1, label %loop.latch
|
|
|
|
loop.latch:
|
|
%l = load i8, ptr %gep.src, align 1
|
|
%xor.red.next = xor i8 %l, %xor.red
|
|
%iv.next = add i64 %iv, 1
|
|
%ec = icmp eq i64 %iv.next, %N
|
|
br i1 %ec, label %exit.2, label %loop.header
|
|
|
|
exit.1:
|
|
ret i64 %x.inc
|
|
|
|
exit.2:
|
|
%r = zext i8 %xor.red.next to i64
|
|
ret i64 %r
|
|
}
|
|
|
|
@gg = constant [80 x i8] c"22 "
|
|
|
|
define i1 @test_exit_compare_other_users() #0 {
|
|
; CHECK-LABEL: define i1 @test_exit_compare_other_users(
|
|
; CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
|
|
; CHECK-NEXT: [[ITER_CHECK:.*]]:
|
|
; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
|
|
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
|
|
; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = sub i64 79, [[INDEX]]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [1 x i8], ptr @gg, i64 [[TMP0]]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 -24
|
|
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -7
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP3]], align 1
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
|
|
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <8 x i8> [[WIDE_LOAD]], <8 x i8> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <8 x i8> [[REVERSE]], splat (i8 32)
|
|
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP4]], i32 7
|
|
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
|
|
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
|
|
; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
|
|
; CHECK: [[VEC_EPILOG_PH]]:
|
|
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 64, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
|
|
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
|
|
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 79, [[INDEX1]]
|
|
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [1 x i8], ptr @gg, i64 [[OFFSET_IDX]]
|
|
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 0
|
|
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 -3
|
|
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1
|
|
; CHECK-NEXT: [[INDEX_NEXT4]] = add nuw i64 [[INDEX1]], 4
|
|
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 76
|
|
; CHECK-NEXT: br i1 [[TMP11]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
|
|
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD2]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <4 x i8> [[REVERSE4]], splat (i8 32)
|
|
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP10]], i32 3
|
|
; CHECK-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
|
|
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
|
|
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 15, %[[VEC_EPILOG_ITER_CHECK]] ], [ 79, %[[ITER_CHECK]] ]
|
|
; CHECK-NEXT: br label %[[LOOP:.*]]
|
|
; CHECK: [[LOOP]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1
|
|
; CHECK-NEXT: [[BOUND_CHECK:%.*]] = icmp ne i64 [[IV]], 0
|
|
; CHECK-NEXT: [[GEP:%.*]] = getelementptr [1 x i8], ptr @gg, i64 [[IV]]
|
|
; CHECK-NEXT: [[LOAD:%.*]] = load i8, ptr [[GEP]], align 1
|
|
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LOAD]], 32
|
|
; CHECK-NEXT: [[CONTINUE:%.*]] = and i1 [[CMP]], [[BOUND_CHECK]]
|
|
; CHECK-NEXT: br i1 [[CONTINUE]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
|
|
; CHECK-NEXT: ret i1 [[CMP_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i64 [ 79, %entry ], [ %iv.next, %loop ]
|
|
%iv.next = add i64 %iv, -1
|
|
%bound.check = icmp ne i64 %iv, 0
|
|
%gep = getelementptr [1 x i8], ptr @gg, i64 %iv
|
|
%load = load i8, ptr %gep, align 1
|
|
%cmp = icmp eq i8 %load, 32
|
|
%continue = and i1 %cmp, %bound.check
|
|
br i1 %continue, label %loop, label %exit
|
|
|
|
exit:
|
|
ret i1 %cmp
|
|
}
|
|
|
|
attributes #0 = { "target-cpu"="znver3" }
|
|
;.
|
|
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
|
|
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
|
|
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
|
|
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
|
|
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
|
|
; CHECK: [[PROF5]] = !{!"branch_weights", i32 4, i32 28}
|
|
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
|
|
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
|
|
;.
|