shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
Florian Hahn 50b9ca4dda
[VPlan] Simplify Plan's entry in removeBranchOnConst. (#154510) 
After https://github.com/llvm/llvm-project/pull/153643, there may be a
BranchOnCond with constant condition in the entry block.

Simplify those in removeBranchOnConst. This removes a number of
redundant conditional branch from entry blocks.

In some cases, it may also make the original scalar loop unreachable,
because we know it will never execute. In that case, we need to remove
the loop from LoopInfo, because all unreachable blocks may dominate each
other, making LoopInfo invalid. In those cases, we can also completely
remove the loop, for which I'll share a follow-up patch.

Depends on https://github.com/llvm/llvm-project/pull/153643.

PR: https://github.com/llvm/llvm-project/pull/154510
2025-09-18 19:25:05 +01:00

1676 lines
103 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --prefix-filecheck-ir-name unnamed --version 5
; RUN: opt %s -S -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 | FileCheck %s
; RUN: opt %s -S -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses | FileCheck %s --check-prefix=INTER
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
%pair = type { i32, i32 }
; Check that a forward consecutive pointer is recognized as uniform and remains
; uniform after vectorization.
define i32 @consecutive_ptr_forward(ptr %a, i64 %n) {
; CHECK-LABEL: define i32 @consecutive_ptr_forward(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP1]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; CHECK-NEXT: [[UNNAMEDTMP2:%.*]] = load i32, ptr [[UNNAMEDTMP1]], align 8
; CHECK-NEXT: [[UNNAMEDTMP3]] = add i32 [[UNNAMEDTMP0]], [[UNNAMEDTMP2]]
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[UNNAMEDTMP4:%.*]] = phi i32 [ [[UNNAMEDTMP3]], %[[FOR_BODY]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[UNNAMEDTMP4]]
;
; INTER-LABEL: define i32 @consecutive_ptr_forward(
; INTER-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; INTER-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
; INTER-NEXT: [[TMP1]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])
; INTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; INTER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; INTER-NEXT: [[UNNAMEDTMP2:%.*]] = load i32, ptr [[UNNAMEDTMP1]], align 8
; INTER-NEXT: [[UNNAMEDTMP3]] = add i32 [[UNNAMEDTMP0]], [[UNNAMEDTMP2]]
; INTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: [[UNNAMEDTMP4:%.*]] = phi i32 [ [[UNNAMEDTMP3]], %[[FOR_BODY]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ]
; INTER-NEXT: ret i32 [[UNNAMEDTMP4]]
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
%tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
%tmp1 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp2 = load i32, ptr %tmp1, align 8
%tmp3 = add i32 %tmp0, %tmp2
%i.next = add nuw nsw i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
%tmp4 = phi i32 [ %tmp3, %for.body ]
ret i32 %tmp4
}
; Check that a reverse consecutive pointer is recognized as uniform and remains
; uniform after vectorization.
define i32 @consecutive_ptr_reverse(ptr %a, i64 %n) {
; CHECK-LABEL: define i32 @consecutive_ptr_reverse(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -1
; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 0
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 -3
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 8
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; CHECK-NEXT: [[TMP5]] = add <4 x i32> [[VEC_PHI]], [[REVERSE]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; CHECK-NEXT: [[UNNAMEDTMP2:%.*]] = load i32, ptr [[UNNAMEDTMP1]], align 8
; CHECK-NEXT: [[UNNAMEDTMP3]] = add i32 [[UNNAMEDTMP0]], [[UNNAMEDTMP2]]
; CHECK-NEXT: [[I_NEXT]] = add nsw i64 [[I]], -1
; CHECK-NEXT: [[COND:%.*]] = icmp sgt i64 [[I_NEXT]], 0
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[UNNAMEDTMP4:%.*]] = phi i32 [ [[UNNAMEDTMP3]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[UNNAMEDTMP4]]
;
; INTER-LABEL: define i32 @consecutive_ptr_reverse(
; INTER-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -1
; INTER-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
; INTER-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; INTER-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[N_VEC]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; INTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[OFFSET_IDX]]
; INTER-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 0
; INTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 -3
; INTER-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP4]], align 8
; INTER-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; INTER-NEXT: [[TMP5]] = add <4 x i32> [[VEC_PHI]], [[REVERSE]]
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
; INTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; INTER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ]
; INTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
; INTER-NEXT: [[UNNAMEDTMP2:%.*]] = load i32, ptr [[UNNAMEDTMP1]], align 8
; INTER-NEXT: [[UNNAMEDTMP3]] = add i32 [[UNNAMEDTMP0]], [[UNNAMEDTMP2]]
; INTER-NEXT: [[I_NEXT]] = add nsw i64 [[I]], -1
; INTER-NEXT: [[COND:%.*]] = icmp sgt i64 [[I_NEXT]], 0
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP5:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: [[UNNAMEDTMP4:%.*]] = phi i32 [ [[UNNAMEDTMP3]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; INTER-NEXT: ret i32 [[UNNAMEDTMP4]]
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %for.body ], [ %n, %entry ]
%tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
%tmp1 = getelementptr inbounds i32, ptr %a, i64 %i
%tmp2 = load i32, ptr %tmp1, align 8
%tmp3 = add i32 %tmp0, %tmp2
%i.next = add nsw i64 %i, -1
%cond = icmp sgt i64 %i.next, 0
br i1 %cond, label %for.body, label %for.end
for.end:
%tmp4 = phi i32 [ %tmp3, %for.body ]
ret i32 %tmp4
}
; Check that a consecutive-like pointer used by a forward interleaved group is
; recognized as uniform and remains uniform after vectorization. When
; interleaved memory accesses aren't enabled, the pointer should not be
; recognized as uniform, and it should not be uniform after vectorization.
define i32 @interleaved_access_forward(ptr %p, i64 %n) {
; CHECK-LABEL: define i32 @interleaved_access_forward(
; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX:%.*]] = add i64 [[INDEX1]], 0
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX1]], 1
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX1]], 2
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[INDEX]], i32 0
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP0]], i32 0
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP1]], i32 0
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP2]], i32 0
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP0]], i32 1
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP1]], i32 1
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP2]], i32 1
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP3]], align 8
; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP4]], align 8
; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP5]], align 8
; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP6]], align 8
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP11]], i32 0
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP12]], i32 1
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP13]], i32 2
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP14]], i32 3
; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP7]], align 8
; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP8]], align 8
; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP9]], align 8
; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP10]], align 8
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> poison, i32 [[TMP19]], i32 0
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP20]], i32 1
; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP21]], i32 2
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP22]], i32 3
; CHECK-NEXT: [[TMP27:%.*]] = add <4 x i32> [[TMP18]], [[TMP26]]
; CHECK-NEXT: [[TMP28]] = add <4 x i32> [[VEC_PHI]], [[TMP27]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP28]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP30]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP6:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0
; CHECK-NEXT: [[UNNAMEDTMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 1
; CHECK-NEXT: [[UNNAMEDTMP3:%.*]] = load i32, ptr [[UNNAMEDTMP1]], align 8
; CHECK-NEXT: [[UNNAMEDTMP4:%.*]] = load i32, ptr [[UNNAMEDTMP2]], align 8
; CHECK-NEXT: [[UNNAMEDTMP5:%.*]] = add i32 [[UNNAMEDTMP3]], [[UNNAMEDTMP4]]
; CHECK-NEXT: [[UNNAMEDTMP6]] = add i32 [[UNNAMEDTMP0]], [[UNNAMEDTMP5]]
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[UNNAMEDTMP14:%.*]] = phi i32 [ [[UNNAMEDTMP6]], %[[FOR_BODY]] ], [ [[TMP30]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[UNNAMEDTMP14]]
;
; INTER-LABEL: define i32 @interleaved_access_forward(
; INTER-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[INDEX]], i32 0
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP0]], align 8
; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; INTER-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; INTER-NEXT: [[TMP1:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
; INTER-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[TMP1]]
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
; INTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; INTER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP6:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0
; INTER-NEXT: [[UNNAMEDTMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 1
; INTER-NEXT: [[UNNAMEDTMP3:%.*]] = load i32, ptr [[UNNAMEDTMP1]], align 8
; INTER-NEXT: [[UNNAMEDTMP4:%.*]] = load i32, ptr [[UNNAMEDTMP2]], align 8
; INTER-NEXT: [[UNNAMEDTMP5:%.*]] = add i32 [[UNNAMEDTMP3]], [[UNNAMEDTMP4]]
; INTER-NEXT: [[UNNAMEDTMP6]] = add i32 [[UNNAMEDTMP0]], [[UNNAMEDTMP5]]
; INTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: [[UNNAMEDTMP14:%.*]] = phi i32 [ [[UNNAMEDTMP6]], %[[FOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
; INTER-NEXT: ret i32 [[UNNAMEDTMP14]]
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
%tmp0 = phi i32 [ %tmp6, %for.body ], [ 0, %entry ]
%tmp1 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 0
%tmp2 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 1
%tmp3 = load i32, ptr %tmp1, align 8
%tmp4 = load i32, ptr %tmp2, align 8
%tmp5 = add i32 %tmp3, %tmp4
%tmp6 = add i32 %tmp0, %tmp5
%i.next = add nuw nsw i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
%tmp14 = phi i32 [ %tmp6, %for.body ]
ret i32 %tmp14
}
; Check that a consecutive-like pointer used by a reverse interleaved group is
; recognized as uniform and remains uniform after vectorization. When
; interleaved memory accesses aren't enabled, the pointer should not be
; recognized as uniform, and it should not be uniform after vectorization.
define i32 @interleaved_access_reverse(ptr %p, i64 %n) {
; CHECK-LABEL: define i32 @interleaved_access_reverse(
; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -1
; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[OFFSET_IDX]], 0
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], -1
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], -2
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], -3
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[TMP18]], i32 0
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP3]], i32 0
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP4]], i32 0
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP5]], i32 0
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP18]], i32 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP3]], i32 1
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP4]], i32 1
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP5]], i32 1
; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP6]], align 8
; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP7]], align 8
; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP8]], align 8
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP9]], align 8
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> poison, i32 [[TMP14]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP15]], i32 1
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP16]], i32 2
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP17]], i32 3
; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP10]], align 8
; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP11]], align 8
; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP12]], align 8
; CHECK-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP13]], align 8
; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 [[TMP22]], i32 0
; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP23]], i32 1
; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP24]], i32 2
; CHECK-NEXT: [[TMP29:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP25]], i32 3
; CHECK-NEXT: [[TMP30:%.*]] = add <4 x i32> [[TMP21]], [[TMP29]]
; CHECK-NEXT: [[TMP31]] = add <4 x i32> [[VEC_PHI]], [[TMP30]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP33:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP31]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP6:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0
; CHECK-NEXT: [[UNNAMEDTMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 1
; CHECK-NEXT: [[UNNAMEDTMP3:%.*]] = load i32, ptr [[UNNAMEDTMP1]], align 8
; CHECK-NEXT: [[UNNAMEDTMP4:%.*]] = load i32, ptr [[UNNAMEDTMP2]], align 8
; CHECK-NEXT: [[UNNAMEDTMP5:%.*]] = add i32 [[UNNAMEDTMP3]], [[UNNAMEDTMP4]]
; CHECK-NEXT: [[UNNAMEDTMP6]] = add i32 [[UNNAMEDTMP0]], [[UNNAMEDTMP5]]
; CHECK-NEXT: [[I_NEXT]] = add nsw i64 [[I]], -1
; CHECK-NEXT: [[COND:%.*]] = icmp sgt i64 [[I_NEXT]], 0
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[UNNAMEDTMP14:%.*]] = phi i32 [ [[UNNAMEDTMP6]], %[[FOR_BODY]] ], [ [[TMP33]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[UNNAMEDTMP14]]
;
; INTER-LABEL: define i32 @interleaved_access_reverse(
; INTER-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -1
; INTER-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
; INTER-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; INTER-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[N_VEC]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; INTER-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[OFFSET_IDX]], i32 0
; INTER-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 0
; INTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 -6
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP4]], align 8
; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; INTER-NEXT: [[VEC_PHI:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; INTER-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; INTER-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
; INTER-NEXT: [[TMP7:%.*]] = add <4 x i32> [[VEC_PHI]], [[TMP6]]
; INTER-NEXT: [[TMP10]] = add <4 x i32> [[VEC_PHI1]], [[TMP7]]
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP10]])
; INTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; INTER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ]
; INTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP6:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0
; INTER-NEXT: [[UNNAMEDTMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 1
; INTER-NEXT: [[UNNAMEDTMP3:%.*]] = load i32, ptr [[UNNAMEDTMP1]], align 8
; INTER-NEXT: [[UNNAMEDTMP4:%.*]] = load i32, ptr [[UNNAMEDTMP2]], align 8
; INTER-NEXT: [[UNNAMEDTMP5:%.*]] = add i32 [[UNNAMEDTMP3]], [[UNNAMEDTMP4]]
; INTER-NEXT: [[UNNAMEDTMP6]] = add i32 [[UNNAMEDTMP0]], [[UNNAMEDTMP5]]
; INTER-NEXT: [[I_NEXT]] = add nsw i64 [[I]], -1
; INTER-NEXT: [[COND:%.*]] = icmp sgt i64 [[I_NEXT]], 0
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP9:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: [[UNNAMEDTMP14:%.*]] = phi i32 [ [[UNNAMEDTMP6]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; INTER-NEXT: ret i32 [[UNNAMEDTMP14]]
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %for.body ], [ %n, %entry ]
%tmp0 = phi i32 [ %tmp6, %for.body ], [ 0, %entry ]
%tmp1 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 0
%tmp2 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 1
%tmp3 = load i32, ptr %tmp1, align 8
%tmp4 = load i32, ptr %tmp2, align 8
%tmp5 = add i32 %tmp3, %tmp4
%tmp6 = add i32 %tmp0, %tmp5
%i.next = add nsw i64 %i, -1
%cond = icmp sgt i64 %i.next, 0
br i1 %cond, label %for.body, label %for.end
for.end:
%tmp14 = phi i32 [ %tmp6, %for.body ]
ret i32 %tmp14
}
; Check that a consecutive-like pointer used by a forward interleaved group and
; scalarized store is not recognized as uniform and is not uniform after
; vectorization. The store is scalarized because it's in a predicated block.
; Even though the load in this example is vectorized and only uses the pointer
; as if it were uniform, the store is scalarized, making the pointer
; non-uniform.
define void @predicated_store(ptr %p, i32 %x, i64 %n) {
; CHECK-LABEL: define void @predicated_store(
; CHECK-SAME: ptr [[P:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[TMP5]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP1]], i32 0
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP12]], i32 0
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP3]], i32 0
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP0]], i32 0
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x ptr> [[TMP13]], ptr [[TMP2]], i32 1
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x ptr> [[TMP19]], ptr [[TMP4]], i32 2
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP25]], ptr [[TMP6]], i32 3
; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP2]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP4]], align 8
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 8
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i32 0
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP8]], i32 1
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP9]], i32 2
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP10]], i32 3
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq <4 x i32> [[TMP14]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP0]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i1> [[TMP15]], i32 1
; CHECK-NEXT: br i1 [[TMP22]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; CHECK: [[PRED_STORE_IF1]]:
; CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP2]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_CONTINUE2]]:
; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i1> [[TMP15]], i32 2
; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; CHECK: [[PRED_STORE_IF3]]:
; CHECK-NEXT: store i32 [[TMP9]], ptr [[TMP4]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; CHECK: [[PRED_STORE_CONTINUE4]]:
; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP15]], i32 3
; CHECK-NEXT: br i1 [[TMP24]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
; CHECK: [[PRED_STORE_IF5]]:
; CHECK-NEXT: store i32 [[TMP10]], ptr [[TMP6]], align 8
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; CHECK: [[PRED_STORE_CONTINUE6]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[IF_MERGE:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0
; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = load i32, ptr [[UNNAMEDTMP0]], align 8
; CHECK-NEXT: [[UNNAMEDTMP2:%.*]] = icmp eq i32 [[UNNAMEDTMP1]], [[X]]
; CHECK-NEXT: br i1 [[UNNAMEDTMP2]], label %[[IF_THEN:.*]], label %[[IF_MERGE]]
; CHECK: [[IF_THEN]]:
; CHECK-NEXT: store i32 [[UNNAMEDTMP1]], ptr [[UNNAMEDTMP0]], align 8
; CHECK-NEXT: br label %[[IF_MERGE]]
; CHECK: [[IF_MERGE]]:
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
; INTER-LABEL: define void @predicated_store(
; INTER-SAME: ptr [[P:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[SMAX]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; INTER-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; INTER-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]]
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[TMP1]]
; INTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; INTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; INTER-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P]], i64 [[INDEX]], i32 0
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 8
; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; INTER-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
; INTER-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
; INTER-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; INTER: [[PRED_STORE_IF]]:
; INTER-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0
; INTER-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP7]], i32 0
; INTER-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i32 0
; INTER-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 8
; INTER-NEXT: br label %[[PRED_STORE_CONTINUE]]
; INTER: [[PRED_STORE_CONTINUE]]:
; INTER-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
; INTER-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; INTER: [[PRED_STORE_IF1]]:
; INTER-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1
; INTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP11]], i32 0
; INTER-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i32 1
; INTER-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 8
; INTER-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; INTER: [[PRED_STORE_CONTINUE2]]:
; INTER-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
; INTER-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; INTER: [[PRED_STORE_IF3]]:
; INTER-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 2
; INTER-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP15]], i32 0
; INTER-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i32 2
; INTER-NEXT: store i32 [[TMP14]], ptr [[TMP13]], align 8
; INTER-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; INTER: [[PRED_STORE_CONTINUE4]]:
; INTER-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
; INTER-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
; INTER: [[PRED_STORE_IF5]]:
; INTER-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], 3
; INTER-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP20]], i32 0
; INTER-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i32 3
; INTER-NEXT: store i32 [[TMP18]], ptr [[TMP17]], align 8
; INTER-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; INTER: [[PRED_STORE_CONTINUE6]]:
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: br label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[IF_MERGE:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0
; INTER-NEXT: [[UNNAMEDTMP1:%.*]] = load i32, ptr [[UNNAMEDTMP0]], align 8
; INTER-NEXT: [[UNNAMEDTMP2:%.*]] = icmp eq i32 [[UNNAMEDTMP1]], [[X]]
; INTER-NEXT: br i1 [[UNNAMEDTMP2]], label %[[IF_THEN:.*]], label %[[IF_MERGE]]
; INTER: [[IF_THEN]]:
; INTER-NEXT: store i32 [[UNNAMEDTMP1]], ptr [[UNNAMEDTMP0]], align 8
; INTER-NEXT: br label %[[IF_MERGE]]
; INTER: [[IF_MERGE]]:
; INTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END:.*]], !llvm.loop [[LOOP11:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: ret void
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ]
%tmp0 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 0
%tmp1 = load i32, ptr %tmp0, align 8
%tmp2 = icmp eq i32 %tmp1, %x
br i1 %tmp2, label %if.then, label %if.merge
if.then:
store i32 %tmp1, ptr %tmp0, align 8
br label %if.merge
if.merge:
%i.next = add nuw nsw i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
ret void
}
; Check that a consecutive pointer used by a scalarized store is not recognized
; as uniform and is not uniform after vectorization. The store is scalarized
; because the stored type may required padding.
define void @irregular_type(ptr %a, i64 %n) {
; CHECK-LABEL: define void @irregular_type(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX:%.*]] = add i64 [[INDEX1]], 0
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX1]], 1
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX1]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX1]], 3
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[TMP5]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[TMP3]]
; CHECK-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP0]], align 16
; CHECK-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP2]], align 16
; CHECK-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP4]], align 16
; CHECK-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP6]], align 16
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = sitofp i32 1 to x86_fp80
; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[I]]
; CHECK-NEXT: store x86_fp80 [[UNNAMEDTMP0]], ptr [[UNNAMEDTMP1]], align 16
; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
; INTER-LABEL: define void @irregular_type(
; INTER-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[INDEX:%.*]] = add i64 [[INDEX1]], 0
; INTER-NEXT: [[TMP1:%.*]] = add i64 [[INDEX1]], 1
; INTER-NEXT: [[TMP5:%.*]] = add i64 [[INDEX1]], 2
; INTER-NEXT: [[TMP3:%.*]] = add i64 [[INDEX1]], 3
; INTER-NEXT: [[TMP0:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[INDEX]]
; INTER-NEXT: [[TMP2:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[TMP1]]
; INTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[TMP5]]
; INTER-NEXT: [[TMP6:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[TMP3]]
; INTER-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP0]], align 16
; INTER-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP2]], align 16
; INTER-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP4]], align 16
; INTER-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[TMP6]], align 16
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
; INTER-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; INTER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = sitofp i32 1 to x86_fp80
; INTER-NEXT: [[UNNAMEDTMP1:%.*]] = getelementptr inbounds x86_fp80, ptr [[A]], i64 [[I]]
; INTER-NEXT: store x86_fp80 [[UNNAMEDTMP0]], ptr [[UNNAMEDTMP1]], align 16
; INTER-NEXT: [[I_NEXT]] = add i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP13:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: ret void
;
entry:
br label %for.body
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = sitofp i32 1 to x86_fp80
%tmp1 = getelementptr inbounds x86_fp80, ptr %a, i64 %i
store x86_fp80 %tmp0, ptr %tmp1, align 16
%i.next = add i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
ret void
}
; Check that a pointer induction variable is recognized as uniform and remains
; uniform after vectorization.
define void @pointer_iv_uniform(ptr %a, i32 %x, i64 %n) {
; CHECK-LABEL: define void @pointer_iv_uniform(
; CHECK-SAME: ptr [[A:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N_VEC]], 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[NEXT_GEP]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP03:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ]
; CHECK-NEXT: store i32 [[X]], ptr [[P]], align 8
; CHECK-NEXT: [[UNNAMEDTMP03]] = getelementptr inbounds i32, ptr [[P]], i32 1
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
; INTER-LABEL: define void @pointer_iv_uniform(
; INTER-SAME: ptr [[A:%.*]], i32 [[X:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; INTER-NEXT: [[TMP0:%.*]] = mul i64 [[N_VEC]], 4
; INTER-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
; INTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
; INTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4
; INTER-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; INTER-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[NEXT_GEP]], align 8
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; INTER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP03:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ]
; INTER-NEXT: store i32 [[X]], ptr [[P]], align 8
; INTER-NEXT: [[UNNAMEDTMP03]] = getelementptr inbounds i32, ptr [[P]], i32 1
; INTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP15:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: ret void
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
%p = phi ptr [ %tmp03, %for.body ], [ %a, %entry ]
store i32 %x, ptr %p, align 8
%tmp03 = getelementptr inbounds i32, ptr %p, i32 1
%i.next = add nuw nsw i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
ret void
}
; Check that a pointer induction variable with a non-uniform user is not
; recognized as uniform and is not uniform after vectorization. The pointer
; induction variable is used by getelementptr instructions that are non-uniform
; due to scalarization of the stores.
define void @pointer_iv_non_uniform_0(ptr %a, i64 %n) {
; CHECK-LABEL: define void @pointer_iv_non_uniform_0(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N_VEC]], 16
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = mul i64 [[INDEX]], 16
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[OFFSET_IDX1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX1]], 16
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX1]], 32
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX1]], 48
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[NEXT_GEP]], align 8
; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[NEXT_GEP1]], align 8
; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[NEXT_GEP2]], align 8
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[NEXT_GEP3]], align 8
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i32 0
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i32 1
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP7]], i32 2
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP8]], i32 3
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 1
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP1]], i32 1
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP2]], i32 1
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP3]], i32 1
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 8
; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 8
; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 8
; CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP16]], align 8
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> poison, i32 [[TMP17]], i32 0
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP18]], i32 1
; CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP19]], i32 2
; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP20]], i32 3
; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 4
; CHECK-NEXT: [[TMP42:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP1]], i32 4
; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP2]], i32 4
; CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP3]], i32 4
; CHECK-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP41]], align 8
; CHECK-NEXT: [[TMP46:%.*]] = load i32, ptr [[TMP42]], align 8
; CHECK-NEXT: [[TMP47:%.*]] = load i32, ptr [[TMP43]], align 8
; CHECK-NEXT: [[TMP56:%.*]] = load i32, ptr [[TMP44]], align 8
; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> poison, i32 [[TMP45]], i32 0
; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP46]], i32 1
; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP47]], i32 2
; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP56]], i32 3
; CHECK-NEXT: [[TMP25:%.*]] = sub <4 x i32> [[TMP24]], [[TMP12]]
; CHECK-NEXT: [[TMP39:%.*]] = sub <4 x i32> [[TMP40]], [[TMP40]]
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 2
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP1]], i32 2
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP2]], i32 2
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP3]], i32 2
; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP25]], i32 0
; CHECK-NEXT: store i32 [[TMP30]], ptr [[TMP26]], align 8
; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP25]], i32 1
; CHECK-NEXT: store i32 [[TMP31]], ptr [[TMP27]], align 8
; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i32> [[TMP25]], i32 2
; CHECK-NEXT: store i32 [[TMP32]], ptr [[TMP28]], align 8
; CHECK-NEXT: [[TMP33:%.*]] = extractelement <4 x i32> [[TMP25]], i32 3
; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP29]], align 8
; CHECK-NEXT: [[TMP48:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 3
; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP1]], i32 3
; CHECK-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP2]], i32 3
; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP3]], i32 3
; CHECK-NEXT: [[TMP52:%.*]] = extractelement <4 x i32> [[TMP39]], i32 0
; CHECK-NEXT: store i32 [[TMP52]], ptr [[TMP48]], align 8
; CHECK-NEXT: [[TMP53:%.*]] = extractelement <4 x i32> [[TMP39]], i32 1
; CHECK-NEXT: store i32 [[TMP53]], ptr [[TMP49]], align 8
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <4 x i32> [[TMP39]], i32 2
; CHECK-NEXT: store i32 [[TMP54]], ptr [[TMP50]], align 8
; CHECK-NEXT: [[TMP55:%.*]] = extractelement <4 x i32> [[TMP39]], i32 3
; CHECK-NEXT: store i32 [[TMP55]], ptr [[TMP51]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP00:%.*]] = load i32, ptr [[P]], align 8
; CHECK-NEXT: [[UNNAMEDTMP03:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
; CHECK-NEXT: [[UNNAMEDTMP04:%.*]] = load i32, ptr [[UNNAMEDTMP03]], align 8
; CHECK-NEXT: [[UNNAMEDTMP3]] = getelementptr inbounds i32, ptr [[P]], i32 4
; CHECK-NEXT: [[UNNAMEDTMP4:%.*]] = load i32, ptr [[UNNAMEDTMP3]], align 8
; CHECK-NEXT: [[UNNAMEDTMP05:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 5
; CHECK-NEXT: [[UNNAMEDTMP06:%.*]] = load i32, ptr [[UNNAMEDTMP05]], align 8
; CHECK-NEXT: [[UNNAMEDTMP07:%.*]] = sub i32 [[UNNAMEDTMP4]], [[UNNAMEDTMP00]]
; CHECK-NEXT: [[UNNAMEDTMP08:%.*]] = sub i32 [[UNNAMEDTMP04]], [[UNNAMEDTMP04]]
; CHECK-NEXT: [[UNNAMEDTMP09:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 2
; CHECK-NEXT: store i32 [[UNNAMEDTMP07]], ptr [[UNNAMEDTMP09]], align 8
; CHECK-NEXT: [[UNNAMEDTMP10:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 3
; CHECK-NEXT: store i32 [[UNNAMEDTMP08]], ptr [[UNNAMEDTMP10]], align 8
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
; INTER-LABEL: define void @pointer_iv_non_uniform_0(
; INTER-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[SMAX]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; INTER-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; INTER-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]]
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[TMP1]]
; INTER-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], 16
; INTER-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[OFFSET_IDX1:%.*]] = mul i64 [[INDEX]], 16
; INTER-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[OFFSET_IDX1]], 0
; INTER-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX1]], 16
; INTER-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX1]], 32
; INTER-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX1]], 48
; INTER-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; INTER-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
; INTER-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
; INTER-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[NEXT_GEP]], align 8
; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; INTER-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; INTER-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 4
; INTER-NEXT: [[WIDE_VEC5:%.*]] = load <16 x i32>, ptr [[TMP7]], align 8
; INTER-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; INTER-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; INTER-NEXT: [[TMP17:%.*]] = sub <4 x i32> [[STRIDED_VEC6]], [[STRIDED_VEC]]
; INTER-NEXT: [[TMP18:%.*]] = sub <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC4]]
; INTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 2
; INTER-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP1]], i32 2
; INTER-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP2]], i32 2
; INTER-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP3]], i32 2
; INTER-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP17]], i32 0
; INTER-NEXT: store i32 [[TMP13]], ptr [[TMP9]], align 8
; INTER-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP17]], i32 1
; INTER-NEXT: store i32 [[TMP14]], ptr [[TMP10]], align 8
; INTER-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[TMP17]], i32 2
; INTER-NEXT: store i32 [[TMP15]], ptr [[TMP11]], align 8
; INTER-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[TMP17]], i32 3
; INTER-NEXT: store i32 [[TMP16]], ptr [[TMP12]], align 8
; INTER-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 3
; INTER-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP1]], i32 3
; INTER-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP2]], i32 3
; INTER-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP3]], i32 3
; INTER-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[TMP18]], i32 0
; INTER-NEXT: store i32 [[TMP23]], ptr [[TMP19]], align 8
; INTER-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP18]], i32 1
; INTER-NEXT: store i32 [[TMP24]], ptr [[TMP20]], align 8
; INTER-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[TMP18]], i32 2
; INTER-NEXT: store i32 [[TMP25]], ptr [[TMP27]], align 8
; INTER-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP18]], i32 3
; INTER-NEXT: store i32 [[TMP26]], ptr [[TMP22]], align 8
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: br label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL8]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP00:%.*]] = load i32, ptr [[P]], align 8
; INTER-NEXT: [[UNNAMEDTMP03:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
; INTER-NEXT: [[UNNAMEDTMP04:%.*]] = load i32, ptr [[UNNAMEDTMP03]], align 8
; INTER-NEXT: [[UNNAMEDTMP3]] = getelementptr inbounds i32, ptr [[P]], i32 4
; INTER-NEXT: [[UNNAMEDTMP4:%.*]] = load i32, ptr [[UNNAMEDTMP3]], align 8
; INTER-NEXT: [[UNNAMEDTMP05:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 5
; INTER-NEXT: [[UNNAMEDTMP06:%.*]] = load i32, ptr [[UNNAMEDTMP05]], align 8
; INTER-NEXT: [[UNNAMEDTMP07:%.*]] = sub i32 [[UNNAMEDTMP4]], [[UNNAMEDTMP00]]
; INTER-NEXT: [[UNNAMEDTMP08:%.*]] = sub i32 [[UNNAMEDTMP04]], [[UNNAMEDTMP04]]
; INTER-NEXT: [[UNNAMEDTMP09:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 2
; INTER-NEXT: store i32 [[UNNAMEDTMP07]], ptr [[UNNAMEDTMP09]], align 8
; INTER-NEXT: [[UNNAMEDTMP10:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 3
; INTER-NEXT: store i32 [[UNNAMEDTMP08]], ptr [[UNNAMEDTMP10]], align 8
; INTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END:.*]], !llvm.loop [[LOOP17:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: ret void
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
%p = phi ptr [ %tmp03, %for.body ], [ %a, %entry ]
%tmp00 = load i32, ptr %p, align 8
%tmp01 = getelementptr inbounds i32, ptr %p, i32 1
%tmp02 = load i32, ptr %tmp01, align 8
%tmp03 = getelementptr inbounds i32, ptr %p, i32 4
%tmp04 = load i32, ptr %tmp03, align 8
%tmp05 = getelementptr inbounds i32, ptr %p, i32 5
%tmp06 = load i32, ptr %tmp05, align 8
%tmp07 = sub i32 %tmp04, %tmp00
%tmp08 = sub i32 %tmp02, %tmp02
%tmp09 = getelementptr inbounds i32, ptr %p, i32 2
store i32 %tmp07, ptr %tmp09, align 8
%tmp10 = getelementptr inbounds i32, ptr %p, i32 3
store i32 %tmp08, ptr %tmp10, align 8
%i.next = add nuw nsw i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
ret void
}
; Check that a pointer induction variable with a non-uniform user is not
; recognized as uniform and is not uniform after vectorization. The pointer
; induction variable is used by a store that will be scalarized.
define void @pointer_iv_non_uniform_1(ptr %a, i64 %n) {
; CHECK-LABEL: define void @pointer_iv_non_uniform_1(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N_VEC]], 16
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = mul i64 [[INDEX]], 16
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[OFFSET_IDX1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX1]], 16
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX1]], 32
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX1]], 48
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
; CHECK-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[NEXT_GEP]], align 16
; CHECK-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[NEXT_GEP1]], align 16
; CHECK-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[NEXT_GEP2]], align 16
; CHECK-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[NEXT_GEP3]], align 16
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP1:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = sitofp i32 1 to x86_fp80
; CHECK-NEXT: store x86_fp80 [[UNNAMEDTMP0]], ptr [[P]], align 16
; CHECK-NEXT: [[UNNAMEDTMP1]] = getelementptr inbounds x86_fp80, ptr [[P]], i32 1
; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
; INTER-LABEL: define void @pointer_iv_non_uniform_1(
; INTER-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; INTER-NEXT: [[TMP0:%.*]] = mul i64 [[N_VEC]], 16
; INTER-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[OFFSET_IDX1:%.*]] = mul i64 [[INDEX]], 16
; INTER-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[OFFSET_IDX1]], 0
; INTER-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX1]], 16
; INTER-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX1]], 32
; INTER-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX1]], 48
; INTER-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
; INTER-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
; INTER-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]]
; INTER-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
; INTER-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[NEXT_GEP]], align 16
; INTER-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[NEXT_GEP1]], align 16
; INTER-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[NEXT_GEP2]], align 16
; INTER-NEXT: store x86_fp80 0xK3FFF8000000000000000, ptr [[NEXT_GEP3]], align 16
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; INTER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTER-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP1:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = sitofp i32 1 to x86_fp80
; INTER-NEXT: store x86_fp80 [[UNNAMEDTMP0]], ptr [[P]], align 16
; INTER-NEXT: [[UNNAMEDTMP1]] = getelementptr inbounds x86_fp80, ptr [[P]], i32 1
; INTER-NEXT: [[I_NEXT]] = add i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP19:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: ret void
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
%p = phi ptr [%tmp1, %for.body], [%a, %entry]
%tmp0 = sitofp i32 1 to x86_fp80
store x86_fp80 %tmp0, ptr %p, align 16
%tmp1 = getelementptr inbounds x86_fp80, ptr %p, i32 1
%i.next = add i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
ret void
}
; Check multiple pointer induction variables where only one is recognized as
; uniform and remains uniform after vectorization. The other pointer induction
; variable is not recognized as uniform and is not uniform after vectorization
; because it is stored to memory.
define i32 @pointer_iv_mixed(ptr %a, ptr %b, i64 %n) {
; CHECK-LABEL: define i32 @pointer_iv_mixed(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 3
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[SMAX]], 2
; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX2]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX2]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], 4
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[VECTOR_GEP]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 8, !alias.scope [[META20:![0-9]+]]
; CHECK-NEXT: [[TMP7]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
; CHECK-NEXT: store <4 x ptr> [[VECTOR_GEP]], ptr [[NEXT_GEP]], align 8, !alias.scope [[META23:![0-9]+]], !noalias [[META20]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ], [ [[A]], %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[B]], %[[ENTRY]] ], [ [[B]], %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[Q:%.*]] = phi ptr [ [[UNNAMEDTMP4:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP2:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; CHECK-NEXT: [[UNNAMEDTMP1:%.*]] = load i32, ptr [[P]], align 8
; CHECK-NEXT: [[UNNAMEDTMP2]] = add i32 [[UNNAMEDTMP1]], [[UNNAMEDTMP0]]
; CHECK-NEXT: store ptr [[P]], ptr [[Q]], align 8
; CHECK-NEXT: [[UNNAMEDTMP3]] = getelementptr inbounds i32, ptr [[P]], i32 1
; CHECK-NEXT: [[UNNAMEDTMP4]] = getelementptr inbounds ptr, ptr [[Q]], i32 1
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[UNNAMEDTMP5:%.*]] = phi i32 [ [[UNNAMEDTMP2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i32 [[UNNAMEDTMP5]]
;
; INTER-LABEL: define i32 @pointer_iv_mixed(
; INTER-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; INTER: [[VECTOR_MEMCHECK]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 3
; INTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
; INTER-NEXT: [[TMP1:%.*]] = shl i64 [[SMAX]], 2
; INTER-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
; INTER-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
; INTER-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
; INTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; INTER-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX2]], 4
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX2]], [[N_MOD_VF]]
; INTER-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], 4
; INTER-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
; INTER-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
; INTER-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
; INTER-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
; INTER-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX]]
; INTER-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[VECTOR_GEP]], i32 0
; INTER-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 8, !alias.scope [[META20:![0-9]+]]
; INTER-NEXT: [[TMP7]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
; INTER-NEXT: store <4 x ptr> [[VECTOR_GEP]], ptr [[NEXT_GEP]], align 8, !alias.scope [[META23:![0-9]+]], !noalias [[META20]]
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16
; INTER-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
; INTER-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; INTER-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; INTER-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ], [ [[A]], %[[VECTOR_MEMCHECK]] ]
; INTER-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[B]], %[[ENTRY]] ], [ [[B]], %[[VECTOR_MEMCHECK]] ]
; INTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[Q:%.*]] = phi ptr [ [[UNNAMEDTMP4:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = phi i32 [ [[UNNAMEDTMP2:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP1:%.*]] = load i32, ptr [[P]], align 8
; INTER-NEXT: [[UNNAMEDTMP2]] = add i32 [[UNNAMEDTMP1]], [[UNNAMEDTMP0]]
; INTER-NEXT: store ptr [[P]], ptr [[Q]], align 8
; INTER-NEXT: [[UNNAMEDTMP3]] = getelementptr inbounds i32, ptr [[P]], i32 1
; INTER-NEXT: [[UNNAMEDTMP4]] = getelementptr inbounds ptr, ptr [[Q]], i32 1
; INTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP26:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: [[UNNAMEDTMP5:%.*]] = phi i32 [ [[UNNAMEDTMP2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; INTER-NEXT: ret i32 [[UNNAMEDTMP5]]
;
entry:
br label %for.body
for.body:
%i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
%p = phi ptr [ %tmp3, %for.body ], [ %a, %entry ]
%q = phi ptr [ %tmp4, %for.body ], [ %b, %entry ]
%tmp0 = phi i32 [ %tmp2, %for.body ], [ 0, %entry ]
%tmp1 = load i32, ptr %p, align 8
%tmp2 = add i32 %tmp1, %tmp0
store ptr %p, ptr %q, align 8
%tmp3 = getelementptr inbounds i32, ptr %p, i32 1
%tmp4 = getelementptr inbounds ptr, ptr %q, i32 1
%i.next = add nuw nsw i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
%tmp5 = phi i32 [ %tmp2, %for.body ]
ret i32 %tmp5
}
; Check that a pointer operand having a user other than a memory access is
; recognized as uniform after vectorization. In this test case, %tmp0 is a
; GEP that is used by a load and a getelementptr instruction (%tmp2). Once
; %tmp2 is marked uniform, %tmp0 should be marked uniform as well.
define void @pointer_operand_geps_with_different_indexed_types(ptr %A, ptr %B, i64 %n) {
; CHECK-LABEL: define void @pointer_operand_geps_with_different_indexed_types(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[SMAX]]
; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[SMAX]], 3
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[TMP6]], -4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[TMP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX2]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX2]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX:%.*]] = add i64 [[INDEX1]], 0
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX1]], 1
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX1]], 2
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX1]], 3
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 3
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 3
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 3
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 3
; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP10]], align 1, !alias.scope [[META27:![0-9]+]]
; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[TMP11]], align 1, !alias.scope [[META27]]
; CHECK-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP12]], align 1, !alias.scope [[META27]]
; CHECK-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP13]], align 1, !alias.scope [[META27]]
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> poison, i8 [[TMP14]], i32 0
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> [[TMP18]], i8 [[TMP15]], i32 1
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP16]], i32 2
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP17]], i32 3
; CHECK-NEXT: [[TMP22:%.*]] = load i8, ptr [[TMP2]], align 1, !alias.scope [[META27]]
; CHECK-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP7]], align 1, !alias.scope [[META27]]
; CHECK-NEXT: [[TMP24:%.*]] = load i8, ptr [[TMP8]], align 1, !alias.scope [[META27]]
; CHECK-NEXT: [[TMP25:%.*]] = load i8, ptr [[TMP9]], align 1, !alias.scope [[META27]]
; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i8> poison, i8 [[TMP22]], i32 0
; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i8> [[TMP26]], i8 [[TMP23]], i32 1
; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x i8> [[TMP27]], i8 [[TMP24]], i32 2
; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i8> [[TMP32]], i8 [[TMP25]], i32 3
; CHECK-NEXT: [[TMP29:%.*]] = xor <4 x i8> [[TMP20]], [[TMP28]]
; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[TMP29]], ptr [[TMP30]], align 1, !alias.scope [[META30:![0-9]+]], !noalias [[META27]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[UNNAMEDTMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
; CHECK-NEXT: [[UNNAMEDTMP2:%.*]] = getelementptr inbounds i8, ptr [[UNNAMEDTMP0]], i64 3
; CHECK-NEXT: [[UNNAMEDTMP3:%.*]] = load i8, ptr [[UNNAMEDTMP2]], align 1
; CHECK-NEXT: [[UNNAMEDTMP4:%.*]] = load i8, ptr [[UNNAMEDTMP0]], align 1
; CHECK-NEXT: [[UNNAMEDTMP5:%.*]] = xor i8 [[UNNAMEDTMP3]], [[UNNAMEDTMP4]]
; CHECK-NEXT: [[UNNAMEDTMP6:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[I]]
; CHECK-NEXT: store i8 [[UNNAMEDTMP5]], ptr [[UNNAMEDTMP6]], align 1
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP33:![0-9]+]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: ret void
;
; INTER-LABEL: define void @pointer_operand_geps_with_different_indexed_types(
; INTER-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; INTER-NEXT: [[ENTRY:.*]]:
; INTER-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[SMAX2]], 4
; INTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; INTER: [[VECTOR_MEMCHECK]]:
; INTER-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; INTER-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[SMAX]]
; INTER-NEXT: [[TMP8:%.*]] = shl i64 [[SMAX]], 3
; INTER-NEXT: [[TMP0:%.*]] = add i64 [[TMP8]], -4
; INTER-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
; INTER-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[TMP1]]
; INTER-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
; INTER-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; INTER-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX2]], 4
; INTER-NEXT: [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; INTER-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]
; INTER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX2]], [[TMP3]]
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; INTER-NEXT: [[WIDE_VEC:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1, !alias.scope [[META27:![0-9]+]]
; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
; INTER-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
; INTER-NEXT: [[TMP5:%.*]] = xor <4 x i8> [[STRIDED_VEC3]], [[STRIDED_VEC]]
; INTER-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
; INTER-NEXT: store <4 x i8> [[TMP5]], ptr [[TMP6]], align 1, !alias.scope [[META30:![0-9]+]], !noalias [[META27]]
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTER-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: br label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
; INTER-NEXT: [[UNNAMEDTMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]]
; INTER-NEXT: [[UNNAMEDTMP2:%.*]] = getelementptr inbounds i8, ptr [[UNNAMEDTMP0]], i64 3
; INTER-NEXT: [[UNNAMEDTMP3:%.*]] = load i8, ptr [[UNNAMEDTMP2]], align 1
; INTER-NEXT: [[UNNAMEDTMP4:%.*]] = load i8, ptr [[UNNAMEDTMP0]], align 1
; INTER-NEXT: [[UNNAMEDTMP5:%.*]] = xor i8 [[UNNAMEDTMP3]], [[UNNAMEDTMP4]]
; INTER-NEXT: [[UNNAMEDTMP6:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[I]]
; INTER-NEXT: store i8 [[UNNAMEDTMP5]], ptr [[UNNAMEDTMP6]], align 1
; INTER-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; INTER-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
; INTER-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END:.*]], !llvm.loop [[LOOP33:![0-9]+]]
; INTER: [[FOR_END]]:
; INTER-NEXT: ret void
;
entry:
br label %for.body
for.body:
%i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
%tmp0 = getelementptr inbounds i64, ptr %A, i64 %i
%tmp2 = getelementptr inbounds i8, ptr %tmp0, i64 3
%tmp3 = load i8, ptr %tmp2, align 1
%tmp4 = load i8, ptr %tmp0, align 1
%tmp5 = xor i8 %tmp3, %tmp4
%tmp6 = getelementptr inbounds i8, ptr %B, i64 %i
store i8 %tmp5, ptr %tmp6
%i.next = add nuw nsw i64 %i, 1
%cond = icmp slt i64 %i.next, %n
br i1 %cond, label %for.body, label %for.end
for.end:
ret void
}
define void @pr61396_pointer_used_as_both_stored_value_and_pointer_operand_by_store(ptr %ary) {
; CHECK-LABEL: define void @pr61396_pointer_used_as_both_stored_value_and_pointer_operand_by_store(
; CHECK-SAME: ptr [[ARY:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds ptr, ptr [[ARY]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x ptr> [[TMP0]], i32 0
; CHECK-NEXT: store <4 x ptr> [[TMP0]], ptr [[TMP1]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 10240, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[CUR_PTR:%.*]] = getelementptr inbounds ptr, ptr [[ARY]], i64 [[IV]]
; CHECK-NEXT: store ptr [[CUR_PTR]], ptr [[CUR_PTR]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV]], 10240
; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP35:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
; INTER-LABEL: define void @pr61396_pointer_used_as_both_stored_value_and_pointer_operand_by_store(
; INTER-SAME: ptr [[ARY:%.*]]) {
; INTER-NEXT: [[ENTRY:.*:]]
; INTER-NEXT: br label %[[VECTOR_PH:.*]]
; INTER: [[VECTOR_PH]]:
; INTER-NEXT: br label %[[VECTOR_BODY:.*]]
; INTER: [[VECTOR_BODY]]:
; INTER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTER-NEXT: [[TMP0:%.*]] = getelementptr inbounds ptr, ptr [[ARY]], <4 x i64> [[VEC_IND]]
; INTER-NEXT: [[TMP1:%.*]] = extractelement <4 x ptr> [[TMP0]], i32 0
; INTER-NEXT: store <4 x ptr> [[TMP0]], ptr [[TMP1]], align 8
; INTER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; INTER-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; INTER-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240
; INTER-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
; INTER: [[MIDDLE_BLOCK]]:
; INTER-NEXT: br label %[[SCALAR_PH:.*]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: br label %[[LOOP:.*]]
; INTER: [[LOOP]]:
; INTER-NEXT: [[IV:%.*]] = phi i64 [ 10240, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; INTER-NEXT: [[CUR_PTR:%.*]] = getelementptr inbounds ptr, ptr [[ARY]], i64 [[IV]]
; INTER-NEXT: store ptr [[CUR_PTR]], ptr [[CUR_PTR]], align 8
; INTER-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; INTER-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV]], 10240
; INTER-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP35:![0-9]+]]
; INTER: [[EXIT]]:
; INTER-NEXT: ret void
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%cur.ptr = getelementptr inbounds ptr, ptr %ary, i64 %iv
store ptr %cur.ptr, ptr %cur.ptr, align 8
%iv.next = add nuw nsw i64 %iv, 1
%done = icmp eq i64 %iv, 10240
br i1 %done, label %exit, label %loop
exit:
ret void
}
Reference in New Issue View Git Blame Copy Permalink