shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
Nikita Popov 573ca36753
[IR] Replace alignment argument with attribute on masked intrinsics (#163802) 
The `masked.load`, `masked.store`, `masked.gather` and `masked.scatter`
intrinsics currently accept a separate alignment immarg. Replace this
with an `align` attribute on the pointer / vector of pointers argument.

This is the standard representation for alignment information on
intrinsics, and is already used by all other memory intrinsics. This
means the signatures now match llvm.expandload, llvm.vp.load, etc.
(Things like llvm.memcpy used to have a separate alignment argument as
well, but were already migrated a long time ago.)

It's worth noting that the masked.gather and masked.scatter intrinsics
previously accepted a zero alignment to indicate the ABI type alignment
of the element type. This special case is gone now: If the align
attribute is omitted, the implied alignment is 1, as usual. If ABI
alignment is desired, it needs to be explicitly emitted (which the
IRBuilder API already requires anyway).
2025-10-20 08:50:09 +00:00

440 lines
16 KiB
LLVM
Raw Blame History

; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s --check-prefixes=COMMON,DEFAULT
; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER
; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER
; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-ENABLE-TP
target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-arm-unknown-eabihf"
; This IR corresponds to this type of C-code:
;
; void f(char *a, char *b, char *c, int N) {
; while (N-- > 0)
; *c++ = *a++ + *b++;
; }
;
define dso_local void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_loopguard(
; COMMON: vector.body:
; CHECK-TF: %[[VIVELEM0:.*]] = extractelement <16 x i32> %vec.iv, i32 0
; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[VIVELEM0]], i32 %N)
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
; CHECK-TF: llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask)
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %while.body.preheader, label %while.end
while.body.preheader:
br label %while.body
while.body:
%N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
%c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
%b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
%a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
%dec = add nsw i32 %N.addr.09, -1
%incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
%0 = load i8, ptr %a.addr.06, align 1
%incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
%1 = load i8, ptr %b.addr.07, align 1
%add = add i8 %1, %0
%incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
store i8 %add, ptr %c.addr.08, align 1
%cmp = icmp sgt i32 %N.addr.09, 1
br i1 %cmp, label %while.body, label %while.end.loopexit
while.end.loopexit:
br label %while.end
while.end:
ret void
}
; No loop-guard: we need one for this to be valid.
;
define dso_local void @sgt_no_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_no_loopguard(
; COMMON: vector.body:
; CHECK-TF: masked.load
; CHECK-TF: masked.load
; CHECK-TF: masked.store
entry:
br label %while.body
while.body:
%N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %entry ]
%c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]
%b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]
%a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry ]
%dec = add nsw i32 %N.addr.09, -1
%incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
%0 = load i8, ptr %a.addr.06, align 1
%incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
%1 = load i8, ptr %b.addr.07, align 1
%add = add i8 %1, %0
%incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
store i8 %add, ptr %c.addr.08, align 1
%cmp = icmp sgt i32 %N.addr.09, 1
br i1 %cmp, label %while.body, label %while.end.loopexit
while.end.loopexit:
br label %while.end
while.end:
ret void
}
define dso_local void @sgt_extra_use_cmp(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_extra_use_cmp(
; COMMON: vector.body:
; CHECK-TF: masked.load
; CHECK-TF: masked.load
; CHECK-TF: masked.store
entry:
br label %while.body
while.body:
%N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %entry ]
%c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]
%b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]
%a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry ]
%dec = add nsw i32 %N.addr.09, -1
%incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
%0 = load i8, ptr %a.addr.06, align 1
%incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
%1 = load i8, ptr %b.addr.07, align 1
%add = add i8 %1, %0
%incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
%cmp = icmp sgt i32 %N.addr.09, 1
%select = select i1 %cmp, i8 %0, i8 %1
%add2 = add i8 %add, %select
store i8 %add2, ptr %c.addr.08, align 1
br i1 %cmp, label %while.body, label %while.end.loopexit
while.end.loopexit:
br label %while.end
while.end:
ret void
}
define dso_local void @sgt_const_tripcount(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_const_tripcount(
; COMMON: vector.body:
; CHECK-TF: masked.load
; CHECK-TF: masked.load
; CHECK-TF: masked.store
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %while.body.preheader, label %while.end
while.body.preheader:
br label %while.body
while.body:
%N.addr.09 = phi i32 [ %dec, %while.body ], [ 2049, %while.body.preheader ]
%c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
%b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
%a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
%dec = add nsw i32 %N.addr.09, -1
%incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
%0 = load i8, ptr %a.addr.06, align 1
%incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
%1 = load i8, ptr %b.addr.07, align 1
%add = add i8 %1, %0
%incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
store i8 %add, ptr %c.addr.08, align 1
%cmp = icmp sgt i32 %N.addr.09, 1
br i1 %cmp, label %while.body, label %while.end.loopexit
while.end.loopexit:
br label %while.end
while.end:
ret void
}
define dso_local void @sgt_no_guard_0_startval(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_no_guard_0_startval(
; COMMON-NOT: vector.body:
entry:
br label %while.body
while.body:
%N.addr.09 = phi i32 [ %dec, %while.body ], [ 0, %entry ]
%c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]
%b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]
%a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry]
%dec = add nsw i32 %N.addr.09, -1
%incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
%0 = load i8, ptr %a.addr.06, align 1
%incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
%1 = load i8, ptr %b.addr.07, align 1
%add = add i8 %1, %0
%incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
store i8 %add, ptr %c.addr.08, align 1
%cmp = icmp sgt i32 %N.addr.09, 1
br i1 %cmp, label %while.body, label %while.end.loopexit
while.end.loopexit:
br label %while.end
while.end:
ret void
}
define dso_local void @sgt_step_minus_two(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_step_minus_two(
; COMMON: vector.body:
; CHECK-TF: masked.load
; CHECK-TF: masked.load
; CHECK-TF: masked.store
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %while.body.preheader, label %while.end
while.body.preheader:
br label %while.body
while.body:
%N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
%c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
%b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
%a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
%dec = add nsw i32 %N.addr.09, -2
%incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
%0 = load i8, ptr %a.addr.06, align 1
%incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
%1 = load i8, ptr %b.addr.07, align 1
%add = add i8 %1, %0
%incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
store i8 %add, ptr %c.addr.08, align 1
%cmp = icmp sgt i32 %N.addr.09, 1
br i1 %cmp, label %while.body, label %while.end.loopexit
while.end.loopexit:
br label %while.end
while.end:
ret void
}
define dso_local void @sgt_step_not_constant(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N, i32 %S) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_step_not_constant(
; COMMON-NOT: vector.body:
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %while.body.preheader, label %while.end
while.body.preheader:
br label %while.body
while.body:
%N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
%c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
%b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
%a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
%dec = add nsw i32 %N.addr.09, %S
%incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
%0 = load i8, ptr %a.addr.06, align 1
%incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
%1 = load i8, ptr %b.addr.07, align 1
%add = add i8 %1, %0
%incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
store i8 %add, ptr %c.addr.08, align 1
%cmp = icmp sgt i32 %N.addr.09, 1
br i1 %cmp, label %while.body, label %while.end.loopexit
while.end.loopexit:
br label %while.end
while.end:
ret void
}
define dso_local void @icmp_eq(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, ptr noalias nocapture %C, i32 %N) #0 {
; COMMON-LABEL: @icmp_eq
; COMMON: vector.body:
entry:
%cmp6 = icmp eq i32 %N, 0
br i1 %cmp6, label %while.end, label %while.body.preheader
while.body.preheader:
br label %while.body
while.body:
%N.addr.010 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
%C.addr.09 = phi ptr [ %incdec.ptr4, %while.body ], [ %C, %while.body.preheader ]
%B.addr.08 = phi ptr [ %incdec.ptr1, %while.body ], [ %B, %while.body.preheader ]
%A.addr.07 = phi ptr [ %incdec.ptr, %while.body ], [ %A, %while.body.preheader ]
%incdec.ptr = getelementptr inbounds i8, ptr %A.addr.07, i32 1
%0 = load i8, ptr %A.addr.07, align 1
%incdec.ptr1 = getelementptr inbounds i8, ptr %B.addr.08, i32 1
%1 = load i8, ptr %B.addr.08, align 1
%add = add i8 %1, %0
%incdec.ptr4 = getelementptr inbounds i8, ptr %C.addr.09, i32 1
store i8 %add, ptr %C.addr.09, align 1
%dec = add i32 %N.addr.010, -1
%cmp = icmp eq i32 %dec, 0
br i1 %cmp, label %while.end.loopexit, label %while.body
while.end.loopexit:
br label %while.end
while.end:
ret void
}
; This IR corresponds to this type of C-code:
;
; void f(char *a, char *b, char * __restrict c, int N) {
; #pragma clang loop vectorize_width(16)
; for (int i = N; i>0; i--)
; c[i] = a[i] + b[i];
; }
;
define dso_local void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_for_loop(
; COMMON: vector.body:
; CHECK-PREFER: masked.load
; CHECK-PREFER: masked.load
; CHECK-PREFER: masked.store
;
; TODO: if tail-predication is requested, tail-folding isn't triggered because
; the profitability check returns "Different strides found, can't tail-predicate",
; investigate this.
;
; CHECK-ENABLE-TP-NOT: masked.load
; CHECK-ENABLE-TP-NOT: masked.load
; CHECK-ENABLE-TP-NOT: masked.store
;
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %for.body.preheader, label %for.end
for.body.preheader:
br label %for.body
for.body:
%i.011 = phi i32 [ %dec, %for.body ], [ %N, %for.body.preheader ]
%arrayidx = getelementptr inbounds i8, ptr %a, i32 %i.011
%0 = load i8, ptr %arrayidx, align 1
%arrayidx1 = getelementptr inbounds i8, ptr %b, i32 %i.011
%1 = load i8, ptr %arrayidx1, align 1
%add = add i8 %1, %0
%arrayidx4 = getelementptr inbounds i8, ptr %c, i32 %i.011
store i8 %add, ptr %arrayidx4, align 1
%dec = add nsw i32 %i.011, -1
%cmp = icmp sgt i32 %i.011, 1
br i1 %cmp, label %for.body, label %for.end, !llvm.loop !1
for.end:
ret void
}
define dso_local void @sgt_for_loop_i64(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_for_loop_i64(
; COMMON: vector.body:
;
; CHECK-PREFER: masked.load
; CHECK-PREFER: masked.load
; CHECK-PREFER: masked.store
;
; With -disable-mve-tail-predication=false, the target hook returns
; "preferPredicateOverEpilogue: hardware-loop is not profitable."
; so here we don't expect the tail-folding. TODO: look into this.
;
; CHECK-ENABLE-TP-NOT: masked.load
; CHECK-ENABLE-TP-NOT: masked.load
; CHECK-ENABLE-TP-NOT: masked.store
;
entry:
%cmp14 = icmp sgt i32 %N, 0
br i1 %cmp14, label %for.body.preheader, label %for.cond.cleanup
for.body.preheader:
%conv16 = zext i32 %N to i64
br label %for.body
for.cond.cleanup.loopexit:
br label %for.cond.cleanup
for.cond.cleanup:
ret void
for.body:
%i.015 = phi i64 [ %dec, %for.body ], [ %conv16, %for.body.preheader ]
%idxprom = trunc i64 %i.015 to i32
%arrayidx = getelementptr inbounds i8, ptr %a, i32 %idxprom
%0 = load i8, ptr %arrayidx, align 1
%arrayidx4 = getelementptr inbounds i8, ptr %b, i32 %idxprom
%1 = load i8, ptr %arrayidx4, align 1
%add = add i8 %1, %0
%arrayidx8 = getelementptr inbounds i8, ptr %c, i32 %idxprom
store i8 %add, ptr %arrayidx8, align 1
%dec = add nsw i64 %i.015, -1
%cmp = icmp sgt i64 %i.015, 1
br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !1
}
; This IR corresponds to this nested-loop:
;
; for (int i = 0; i<N; i++)
; for (int j = i+1; j>0; j--)
; c[j] = a[j] + b[j];
;
; while the inner-loop looks similar to previous examples, we can't
; transform this because the inner loop because isGuarded returns
; false for the inner-loop.
;
define dso_local void @sgt_nested_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {
; COMMON-LABEL: @sgt_nested_loop(
; DEFAULT-NOT: vector.body:
; CHECK-TF-NOT: masked.load
; CHECK-TF-NOT: masked.load
; CHECK-TF-NOT: masked.store
; COMMON: }
;
entry:
%cmp21 = icmp sgt i32 %N, 0
br i1 %cmp21, label %for.body.preheader, label %for.cond.cleanup
for.body.preheader:
br label %for.body
for.cond.loopexit:
%exitcond = icmp eq i32 %add, %N
br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
for.cond.cleanup.loopexit:
br label %for.cond.cleanup
for.cond.cleanup:
ret void
for.body:
%i.022 = phi i32 [ %add, %for.cond.loopexit ], [ 0, %for.body.preheader ]
%add = add nuw nsw i32 %i.022, 1
br label %for.body4
for.body4: ; preds = %for.body, %for.body4
%j.020 = phi i32 [ %add, %for.body ], [ %dec, %for.body4 ]
%arrayidx = getelementptr inbounds i8, ptr %a, i32 %j.020
%0 = load i8, ptr %arrayidx, align 1
%arrayidx5 = getelementptr inbounds i8, ptr %b, i32 %j.020
%1 = load i8, ptr %arrayidx5, align 1
%add7 = add i8 %1, %0
%arrayidx9 = getelementptr inbounds i8, ptr %c, i32 %j.020
store i8 %add7, ptr %arrayidx9, align 1
%dec = add nsw i32 %j.020, -1
%cmp2 = icmp sgt i32 %j.020, 1
br i1 %cmp2, label %for.body4, label %for.cond.loopexit
}
attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }
!1 = distinct !{!1, !2}
!2 = !{!"llvm.loop.vectorize.width", i32 16}
Reference in New Issue View Git Blame Copy Permalink