fa3ec0c17c
Materialize constant vector trip counts before ::execute, if the trip count can be computed as Original (TC / (VF * UF)) * (VF * UF). For now this excludes when the tail is folded or scalar epilogues are required. This enables removing a number of redundant branches from the middle block. For now this is also only done when not vectorizing the epilogue, as the simplification complicates stitching the 2 plans together. PR: https://github.com/llvm/llvm-project/pull/142309
157 lines
7.8 KiB
LLVM
157 lines
7.8 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br" --filter "^.*:"
|
|
; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
|
|
; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=4 -S < %s | FileCheck %s -check-prefix=CHECK-MASKED
|
|
; RUN: opt -passes="print<block-freq>,loop-vectorize" -force-vector-width=4 -force-vector-interleave=1 \
|
|
; RUN: -scalable-vectorization=on -force-target-supports-scalable-vectors -S < %s | FileCheck %s -check-prefix=CHECK-SCALABLE
|
|
|
|
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
|
|
|
|
@a = global [1024 x i32] zeroinitializer, align 16
|
|
@b = global [1024 x i32] zeroinitializer, align 16
|
|
|
|
; Check correctness of profile info for vectorization without epilog.
|
|
define void @_Z3foov() {
|
|
; CHECK-LABEL: @_Z3foov(
|
|
; CHECK: entry:
|
|
; CHECK: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0:![0-9]+]]
|
|
; CHECK: vector.ph:
|
|
; CHECK: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK: vector.body:
|
|
; CHECK: br i1 [[TMP6:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
|
|
; CHECK: middle.block:
|
|
; CHECK: br label [[FOR_COND_CLEANUP:%.*]]
|
|
; CHECK: scalar.ph:
|
|
; CHECK: br label [[FOR_BODY:%.*]]
|
|
; CHECK: for.cond.cleanup:
|
|
; CHECK: for.body:
|
|
; CHECK: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF5:![0-9]+]], !llvm.loop [[LOOP6:![0-9]+]]
|
|
;
|
|
; CHECK-MASKED-LABEL: @_Z3foov(
|
|
; CHECK-MASKED: entry:
|
|
; CHECK-MASKED: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0:![0-9]+]]
|
|
; CHECK-MASKED: vector.ph:
|
|
; CHECK-MASKED: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK-MASKED: vector.body:
|
|
; CHECK-MASKED: br i1 [[TMP18:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
|
|
; CHECK-MASKED: middle.block:
|
|
; CHECK-MASKED: br label [[FOR_COND_CLEANUP:%.*]]
|
|
; CHECK-MASKED: scalar.ph:
|
|
; CHECK-MASKED: br label [[FOR_BODY:%.*]]
|
|
; CHECK-MASKED: for.cond.cleanup:
|
|
; CHECK-MASKED: for.body:
|
|
; CHECK-MASKED: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF5:![0-9]+]], !llvm.loop [[LOOP6:![0-9]+]]
|
|
;
|
|
; CHECK-SCALABLE-LABEL: @_Z3foov(
|
|
; CHECK-SCALABLE: entry:
|
|
; CHECK-SCALABLE: br i1 [[MIN_ITERS_CHECK:%.*]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0:![0-9]+]]
|
|
; CHECK-SCALABLE: vector.ph:
|
|
; CHECK-SCALABLE: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9:%.*]], i64 0
|
|
; CHECK-SCALABLE: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
|
|
; CHECK-SCALABLE: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK-SCALABLE: vector.body:
|
|
; CHECK-SCALABLE: [[VEC_IND_NEXT:%.*]] = add <vscale x 4 x i32> [[VEC_IND:%.*]], [[BROADCAST_SPLAT]]
|
|
; CHECK-SCALABLE: br i1 [[TMP16:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
|
|
; CHECK-SCALABLE: middle.block:
|
|
; CHECK-SCALABLE: br i1 [[CMP_N:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF5:![0-9]+]]
|
|
; CHECK-SCALABLE: scalar.ph:
|
|
; CHECK-SCALABLE: br label [[FOR_BODY:%.*]]
|
|
; CHECK-SCALABLE: for.cond.cleanup:
|
|
; CHECK-SCALABLE: for.body:
|
|
; CHECK-SCALABLE: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.cond.cleanup: ; preds = %for.body
|
|
ret void
|
|
|
|
for.body: ; preds = %for.body, %entry
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
|
|
%arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %iv
|
|
%0 = load i32, ptr %arrayidx, align 4
|
|
%1 = trunc i64 %iv to i32
|
|
%mul = mul nsw i32 %0, %1
|
|
%arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %iv
|
|
%2 = load i32, ptr %arrayidx2, align 4
|
|
%add = add nsw i32 %2, %mul
|
|
store i32 %add, ptr %arrayidx2, align 4
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%exitcond = icmp eq i64 %iv.next, 1024
|
|
br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !0
|
|
}
|
|
|
|
; Check correctness of profile info for vectorization with epilog.
|
|
define void @_Z3foo2v() {
|
|
; CHECK-LABEL: @_Z3foo2v(
|
|
; CHECK: entry:
|
|
; CHECK: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0]]
|
|
; CHECK: vector.ph:
|
|
; CHECK: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK: vector.body:
|
|
; CHECK: br i1 [[TMP6:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP7:![0-9]+]]
|
|
; CHECK: middle.block:
|
|
; CHECK: br label [[SCALAR_PH]]
|
|
; CHECK: scalar.ph:
|
|
; CHECK: br label [[FOR_BODY:%.*]]
|
|
; CHECK: for.cond.cleanup:
|
|
; CHECK: for.body:
|
|
; CHECK: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
|
|
;
|
|
; CHECK-MASKED-LABEL: @_Z3foo2v(
|
|
; CHECK-MASKED: entry:
|
|
; CHECK-MASKED: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0]]
|
|
; CHECK-MASKED: vector.ph:
|
|
; CHECK-MASKED: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK-MASKED: vector.body:
|
|
; CHECK-MASKED: br i1 [[TMP18:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP7:![0-9]+]]
|
|
; CHECK-MASKED: middle.block:
|
|
; CHECK-MASKED: br label [[SCALAR_PH]]
|
|
; CHECK-MASKED: scalar.ph:
|
|
; CHECK-MASKED: br label [[FOR_BODY:%.*]]
|
|
; CHECK-MASKED: for.cond.cleanup:
|
|
; CHECK-MASKED: for.body:
|
|
; CHECK-MASKED: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
|
|
;
|
|
; CHECK-SCALABLE-LABEL: @_Z3foo2v(
|
|
; CHECK-SCALABLE: entry:
|
|
; CHECK-SCALABLE: br i1 [[MIN_ITERS_CHECK:%.*]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !prof [[PROF0]]
|
|
; CHECK-SCALABLE: vector.ph:
|
|
; CHECK-SCALABLE: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9:%.*]], i64 0
|
|
; CHECK-SCALABLE: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
|
|
; CHECK-SCALABLE: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK-SCALABLE: vector.body:
|
|
; CHECK-SCALABLE: [[VEC_IND_NEXT:%.*]] = add <vscale x 4 x i32> [[VEC_IND:%.*]], [[BROADCAST_SPLAT]]
|
|
; CHECK-SCALABLE: br i1 [[TMP16:%.*]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP8:![0-9]+]]
|
|
; CHECK-SCALABLE: middle.block:
|
|
; CHECK-SCALABLE: br i1 [[CMP_N:%.*]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]], !prof [[PROF5]]
|
|
; CHECK-SCALABLE: scalar.ph:
|
|
; CHECK-SCALABLE: br label [[FOR_BODY:%.*]]
|
|
; CHECK-SCALABLE: for.cond.cleanup:
|
|
; CHECK-SCALABLE: for.body:
|
|
; CHECK-SCALABLE: br i1 [[EXITCOND:%.*]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.cond.cleanup: ; preds = %for.body
|
|
ret void
|
|
|
|
for.body: ; preds = %for.body, %entry
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
|
|
%arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %iv
|
|
%0 = load i32, ptr %arrayidx, align 4
|
|
%1 = trunc i64 %iv to i32
|
|
%mul = mul nsw i32 %0, %1
|
|
%arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %iv
|
|
%2 = load i32, ptr %arrayidx2, align 4
|
|
%add = add nsw i32 %2, %mul
|
|
store i32 %add, ptr %arrayidx2, align 4
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%exitcond = icmp eq i64 %iv.next, 1027
|
|
br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !1
|
|
}
|
|
|
|
|
|
!0 = !{!"branch_weights", i32 1, i32 1023}
|
|
!1 = !{!"branch_weights", i32 1, i32 1026}
|