e844f05397
This simplifies the return value of addRuntimeCheck from a pair of instructions to a single `Value *`. The existing users of addRuntimeChecks were ignoring the first element of the pair, hence there is not reason to track FirstInst and return it. Additionally all users of addRuntimeChecks use the second returned `Instruction *` just as `Value *`, so there is no need to return an `Instruction *`. Therefore there is no need to create a redundant dummy `and X, true` instruction any longer. Effectively this change should not impact the generated code because the redundant AND will be folded by later optimizations. But it is easy to avoid creating it in the first place and it allows more accurately estimating the cost of the runtime checks.
140 lines
7.5 KiB
LLVM
140 lines
7.5 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
|
|
; RUN: opt -loop-vectorize -force-vector-width=4 -S < %s | FileCheck %s
|
|
|
|
; This is the test case from PR26314.
|
|
; When we were retrying dependence checking with memchecks only,
|
|
; the loop-invariant access in the inner loop was incorrectly determined to be wrapping
|
|
; because it was not strided in the inner loop.
|
|
; Improved wrapping detection allows vectorization in the following case.
|
|
|
|
; #define Z 32
|
|
; typedef struct s {
|
|
; int v1[Z];
|
|
; int v2[Z];
|
|
; int v3[Z][Z];
|
|
; } s;
|
|
;
|
|
; void slow_function (s* const obj, int z) {
|
|
; for (int j=0; j<Z; j++) {
|
|
; for (int k=0; k<z; k++) {
|
|
; int x = obj->v1[k] + obj->v2[j];
|
|
; obj->v3[j][k] += x;
|
|
; }
|
|
; }
|
|
; }
|
|
|
|
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
|
|
|
|
%struct.s = type { [32 x i32], [32 x i32], [32 x [32 x i32]] }
|
|
|
|
define void @Test(%struct.s* nocapture %obj, i64 %z) #0 {
|
|
; CHECK-LABEL: @Test(
|
|
; CHECK-NEXT: [[OBJ4:%.*]] = bitcast %struct.s* [[OBJ:%.*]] to i8*
|
|
; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr [[STRUCT_S:%.*]], %struct.s* [[OBJ]], i64 0, i32 0, i64 [[Z:%.*]]
|
|
; CHECK-NEXT: [[SCEVGEP56:%.*]] = bitcast i32* [[SCEVGEP5]] to i8*
|
|
; CHECK-NEXT: br label [[DOTOUTER_PREHEADER:%.*]]
|
|
; CHECK: .outer.preheader:
|
|
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[I_NEXT:%.*]], [[DOTOUTER:%.*]] ]
|
|
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 2, i64 [[I]], i64 0
|
|
; CHECK-NEXT: [[SCEVGEP1:%.*]] = bitcast i32* [[SCEVGEP]] to i8*
|
|
; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 2, i64 [[I]], i64 [[Z]]
|
|
; CHECK-NEXT: [[SCEVGEP23:%.*]] = bitcast i32* [[SCEVGEP2]] to i8*
|
|
; CHECK-NEXT: [[SCEVGEP7:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 1, i64 [[I]]
|
|
; CHECK-NEXT: [[SCEVGEP78:%.*]] = bitcast i32* [[SCEVGEP7]] to i8*
|
|
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[I]], 1
|
|
; CHECK-NEXT: [[SCEVGEP9:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 1, i64 [[TMP1]]
|
|
; CHECK-NEXT: [[SCEVGEP910:%.*]] = bitcast i32* [[SCEVGEP9]] to i8*
|
|
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 1, i64 [[I]]
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[Z]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
|
|
; CHECK: vector.memcheck:
|
|
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP56]]
|
|
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[OBJ4]], [[SCEVGEP23]]
|
|
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
|
|
; CHECK-NEXT: [[BOUND011:%.*]] = icmp ult i8* [[SCEVGEP1]], [[SCEVGEP910]]
|
|
; CHECK-NEXT: [[BOUND112:%.*]] = icmp ult i8* [[SCEVGEP78]], [[SCEVGEP23]]
|
|
; CHECK-NEXT: [[FOUND_CONFLICT13:%.*]] = and i1 [[BOUND011]], [[BOUND112]]
|
|
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT13]]
|
|
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
|
|
; CHECK: vector.ph:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[Z]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[Z]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
|
|
; CHECK: vector.body:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0
|
|
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 0, i64 [[TMP3]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
|
|
; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4, !alias.scope !0
|
|
; CHECK-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP2]], align 4, !alias.scope !3
|
|
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i32 0
|
|
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 2, i64 [[I]], i64 [[TMP3]]
|
|
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
|
|
; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
|
|
; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4, !alias.scope !5, !noalias !7
|
|
; CHECK-NEXT: [[TMP12:%.*]] = add nsw <4 x i32> [[TMP8]], [[WIDE_LOAD14]]
|
|
; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32* [[TMP10]] to <4 x i32>*
|
|
; CHECK-NEXT: store <4 x i32> [[TMP12]], <4 x i32>* [[TMP13]], align 4, !alias.scope !5, !noalias !7
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
|
|
; CHECK: middle.block:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[Z]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], label [[DOTOUTER]], label [[SCALAR_PH]]
|
|
; CHECK: scalar.ph:
|
|
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTOUTER_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
|
|
; CHECK-NEXT: br label [[DOTINNER:%.*]]
|
|
; CHECK: .exit:
|
|
; CHECK-NEXT: ret void
|
|
; CHECK: .outer:
|
|
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
|
|
; CHECK-NEXT: [[EXITCOND_OUTER:%.*]] = icmp eq i64 [[I_NEXT]], 32
|
|
; CHECK-NEXT: br i1 [[EXITCOND_OUTER]], label [[DOTEXIT:%.*]], label [[DOTOUTER_PREHEADER]]
|
|
; CHECK: .inner:
|
|
; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[J_NEXT:%.*]], [[DOTINNER]] ]
|
|
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 0, i64 [[J]]
|
|
; CHECK-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP15]], align 4
|
|
; CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP2]], align 4
|
|
; CHECK-NEXT: [[TMP18:%.*]] = add nsw i32 [[TMP17]], [[TMP16]]
|
|
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[OBJ]], i64 0, i32 2, i64 [[I]], i64 [[J]]
|
|
; CHECK-NEXT: [[TMP20:%.*]] = load i32, i32* [[TMP19]], align 4
|
|
; CHECK-NEXT: [[TMP21:%.*]] = add nsw i32 [[TMP18]], [[TMP20]]
|
|
; CHECK-NEXT: store i32 [[TMP21]], i32* [[TMP19]], align 4
|
|
; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
|
|
; CHECK-NEXT: [[EXITCOND_INNER:%.*]] = icmp eq i64 [[J_NEXT]], [[Z]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND_INNER]], label [[DOTOUTER]], label [[DOTINNER]], !llvm.loop [[LOOP10:![0-9]+]]
|
|
;
|
|
br label %.outer.preheader
|
|
|
|
|
|
.outer.preheader:
|
|
%i = phi i64 [ 0, %0 ], [ %i.next, %.outer ]
|
|
%1 = getelementptr inbounds %struct.s, %struct.s* %obj, i64 0, i32 1, i64 %i
|
|
br label %.inner
|
|
|
|
.exit:
|
|
ret void
|
|
|
|
.outer:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%exitcond.outer = icmp eq i64 %i.next, 32
|
|
br i1 %exitcond.outer, label %.exit, label %.outer.preheader
|
|
|
|
.inner:
|
|
%j = phi i64 [ 0, %.outer.preheader ], [ %j.next, %.inner ]
|
|
%2 = getelementptr inbounds %struct.s, %struct.s* %obj, i64 0, i32 0, i64 %j
|
|
%3 = load i32, i32* %2
|
|
%4 = load i32, i32* %1
|
|
%5 = add nsw i32 %4, %3
|
|
%6 = getelementptr inbounds %struct.s, %struct.s* %obj, i64 0, i32 2, i64 %i, i64 %j
|
|
%7 = load i32, i32* %6
|
|
%8 = add nsw i32 %5, %7
|
|
store i32 %8, i32* %6
|
|
%j.next = add nuw nsw i64 %j, 1
|
|
%exitcond.inner = icmp eq i64 %j.next, %z
|
|
br i1 %exitcond.inner, label %.outer, label %.inner
|
|
}
|