shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/test/Transforms/LoopVectorize/RISCV/runtime-check-dependent-on-stride.ll
Andrei Elovikov 7018ddfb98
[NFC][VPlan] Test showing that unit-stride-mv should be done later in pipeline (#180292) 
Right now memory dependencies checks and speculation for unit-strideness
are performed somewhat simultaneously. This is wrong because:

* Ideally, if accesses aren't unit-strided in runtime we might want to
take a version with gather/strided load (longer term). Those two loops
should share legality checks and the dispatch based on stride should
only happen after the legality condition has been satisfied.
* Even if we don't generate multiple vector loops (current situation),
not vectorizing at all is worse than generating gather-only vector loop.

This PR adds a test for the latter as that could be a first step in
adding full support for the former.

This isn't target-specific, but gathers aren't supported in generic
target and result in very ugly scalarized code/CHECKs, hence put the
test under RISCV/.

Co-authored-by: Florian Hahn <flo@fhahn.com>
2026-02-17 16:24:51 +00:00

159 lines
12 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -mtriple=riscv64 -mattr=+v -passes=loop-vectorize -S < %s -enable-mem-access-versioning=true | FileCheck %s --check-prefix=UNIT-STRIDE-MV
; RUN: opt -mtriple=riscv64 -mattr=+v -passes=loop-vectorize -S < %s -enable-mem-access-versioning=false | FileCheck %s --check-prefix=NO-UNIT-STRIDE-MV
; TODO: Make sure that optimizations (unit-stride multiversioning) don't prohibit vectorization.
define void @foo(ptr %p, ptr %p.strided, i64 %n, i64 %stride) {
; UNIT-STRIDE-MV-LABEL: define void @foo(
; UNIT-STRIDE-MV-SAME: ptr [[P:%.*]], ptr [[P_STRIDED:%.*]], i64 [[N:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
; UNIT-STRIDE-MV-NEXT: [[ENTRY:.*]]:
; UNIT-STRIDE-MV-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[STRIDE]], 2
; UNIT-STRIDE-MV-NEXT: [[MUL:%.*]] = mul nuw nsw i64 [[ADD]], [[ADD]]
; UNIT-STRIDE-MV-NEXT: [[OUT_OFFSET:%.*]] = add i64 [[MUL]], 16
; UNIT-STRIDE-MV-NEXT: [[OUT:%.*]] = getelementptr i64, ptr [[P]], i64 [[OUT_OFFSET]]
; UNIT-STRIDE-MV-NEXT: br label %[[HEADER:.*]]
; UNIT-STRIDE-MV: [[HEADER]]:
; UNIT-STRIDE-MV-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; UNIT-STRIDE-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; UNIT-STRIDE-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; UNIT-STRIDE-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
; UNIT-STRIDE-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[OUT]], i64 [[IV]]
; UNIT-STRIDE-MV-NEXT: [[GEP_STRIDED:%.*]] = getelementptr i64, ptr [[P_STRIDED]], i64 [[IDX]]
; UNIT-STRIDE-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD]], align 8
; UNIT-STRIDE-MV-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_STRIDED]], align 8
; UNIT-STRIDE-MV-NEXT: [[LD:%.*]] = add i64 [[LD1]], [[LD2]]
; UNIT-STRIDE-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; UNIT-STRIDE-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 64
; UNIT-STRIDE-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; UNIT-STRIDE-MV: [[EXIT]]:
; UNIT-STRIDE-MV-NEXT: ret void
;
; NO-UNIT-STRIDE-MV-LABEL: define void @foo(
; NO-UNIT-STRIDE-MV-SAME: ptr [[P:%.*]], ptr [[P_STRIDED:%.*]], i64 [[N:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0:[0-9]+]] {
; NO-UNIT-STRIDE-MV-NEXT: [[ENTRY:.*:]]
; NO-UNIT-STRIDE-MV-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[STRIDE]], 2
; NO-UNIT-STRIDE-MV-NEXT: [[MUL:%.*]] = mul nuw nsw i64 [[ADD]], [[ADD]]
; NO-UNIT-STRIDE-MV-NEXT: [[OUT_OFFSET:%.*]] = add i64 [[MUL]], 16
; NO-UNIT-STRIDE-MV-NEXT: [[OUT:%.*]] = getelementptr i64, ptr [[P]], i64 [[OUT_OFFSET]]
; NO-UNIT-STRIDE-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; NO-UNIT-STRIDE-MV: [[VECTOR_SCEVCHECK]]:
; NO-UNIT-STRIDE-MV-NEXT: [[TMP2:%.*]] = shl i64 [[STRIDE]], 3
; NO-UNIT-STRIDE-MV-NEXT: [[TMP3:%.*]] = mul i64 [[STRIDE]], -8
; NO-UNIT-STRIDE-MV-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P_STRIDED]], i64 [[TMP2]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP4:%.*]] = icmp slt i64 [[TMP2]], 0
; NO-UNIT-STRIDE-MV-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[TMP2]]
; NO-UNIT-STRIDE-MV-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[TMP5]], i64 62)
; NO-UNIT-STRIDE-MV-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
; NO-UNIT-STRIDE-MV-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
; NO-UNIT-STRIDE-MV-NEXT: [[TMP6:%.*]] = sub i64 0, [[MUL_RESULT]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP6]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP9:%.*]] = icmp ult ptr [[TMP7]], [[SCEVGEP]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP10:%.*]] = icmp ugt ptr [[TMP8]], [[SCEVGEP]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP11:%.*]] = select i1 [[TMP4]], i1 [[TMP10]], i1 [[TMP9]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[TMP12]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; NO-UNIT-STRIDE-MV: [[VECTOR_MEMCHECK]]:
; NO-UNIT-STRIDE-MV-NEXT: [[TMP13:%.*]] = mul i64 [[ADD]], [[ADD]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP14:%.*]] = shl i64 [[TMP13]], 3
; NO-UNIT-STRIDE-MV-NEXT: [[TMP15:%.*]] = add i64 [[TMP14]], 136
; NO-UNIT-STRIDE-MV-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP15]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP18:%.*]] = add i64 [[TMP14]], 640
; NO-UNIT-STRIDE-MV-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP18]]
; NO-UNIT-STRIDE-MV-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[P]], i64 8
; NO-UNIT-STRIDE-MV-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[P]], i64 512
; NO-UNIT-STRIDE-MV-NEXT: [[TMP20:%.*]] = mul i64 [[STRIDE]], 504
; NO-UNIT-STRIDE-MV-NEXT: [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[P_STRIDED]], i64 [[TMP20]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP21:%.*]] = shl i64 [[STRIDE]], 3
; NO-UNIT-STRIDE-MV-NEXT: [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[P_STRIDED]], i64 [[TMP21]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP22:%.*]] = icmp ult ptr [[SCEVGEP7]], [[SCEVGEP8]]
; NO-UNIT-STRIDE-MV-NEXT: [[UMIN:%.*]] = select i1 [[TMP22]], ptr [[SCEVGEP7]], ptr [[SCEVGEP8]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP23:%.*]] = icmp ugt ptr [[SCEVGEP7]], [[SCEVGEP8]]
; NO-UNIT-STRIDE-MV-NEXT: [[UMAX:%.*]] = select i1 [[TMP23]], ptr [[SCEVGEP7]], ptr [[SCEVGEP8]]
; NO-UNIT-STRIDE-MV-NEXT: [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[UMAX]], i64 8
; NO-UNIT-STRIDE-MV-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP6]]
; NO-UNIT-STRIDE-MV-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP5]], [[SCEVGEP4]]
; NO-UNIT-STRIDE-MV-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; NO-UNIT-STRIDE-MV-NEXT: [[BOUND010:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP9]]
; NO-UNIT-STRIDE-MV-NEXT: [[BOUND111:%.*]] = icmp ult ptr [[UMIN]], [[SCEVGEP4]]
; NO-UNIT-STRIDE-MV-NEXT: [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]]
; NO-UNIT-STRIDE-MV-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]]
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; NO-UNIT-STRIDE-MV: [[VECTOR_PH]]:
; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[STRIDE]], i64 0
; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT15]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; NO-UNIT-STRIDE-MV-NEXT: [[TMP35:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; NO-UNIT-STRIDE-MV-NEXT: [[INDUCTION:%.*]] = add nsw <vscale x 2 x i64> splat (i64 1), [[TMP35]]
; NO-UNIT-STRIDE-MV-NEXT: br label %[[HEADER:.*]]
; NO-UNIT-STRIDE-MV: [[HEADER]]:
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[STEP_ADD_3:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL:%.*]] = phi i64 [ 63, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[HEADER]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP25:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
; NO-UNIT-STRIDE-MV-NEXT: [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP26]], i64 0
; NO-UNIT-STRIDE-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT14]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; NO-UNIT-STRIDE-MV-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP34:%.*]] = mul <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT16]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET_IDX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP31:%.*]] = getelementptr i64, ptr [[OUT]], i64 [[OFFSET_IDX]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP40:%.*]] = getelementptr i64, ptr [[P_STRIDED]], <vscale x 2 x i64> [[TMP34]]
; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_LOAD19:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP30]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META0:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[WIDE_MASKED_GATHER22:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP40]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META3:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP49:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD19]], [[WIDE_MASKED_GATHER22]]
; NO-UNIT-STRIDE-MV-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP49]], ptr align 8 [[TMP31]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP25]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
; NO-UNIT-STRIDE-MV-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP26]], [[INDEX]]
; NO-UNIT-STRIDE-MV-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
; NO-UNIT-STRIDE-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]
; NO-UNIT-STRIDE-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
; NO-UNIT-STRIDE-MV: [[MIDDLE_BLOCK]]:
; NO-UNIT-STRIDE-MV-NEXT: br label %[[EXIT:.*]]
; NO-UNIT-STRIDE-MV: [[SCALAR_PH]]:
; NO-UNIT-STRIDE-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, %[[VECTOR_SCEVCHECK]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
; NO-UNIT-STRIDE-MV-NEXT: br label %[[HEADER1:.*]]
; NO-UNIT-STRIDE-MV: [[HEADER1]]:
; NO-UNIT-STRIDE-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; NO-UNIT-STRIDE-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; NO-UNIT-STRIDE-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; NO-UNIT-STRIDE-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
; NO-UNIT-STRIDE-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[OUT]], i64 [[IV]]
; NO-UNIT-STRIDE-MV-NEXT: [[GEP_STRIDED:%.*]] = getelementptr i64, ptr [[P_STRIDED]], i64 [[IDX]]
; NO-UNIT-STRIDE-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD]], align 8
; NO-UNIT-STRIDE-MV-NEXT: [[LD2:%.*]] = load i64, ptr [[GEP_STRIDED]], align 8
; NO-UNIT-STRIDE-MV-NEXT: [[LD:%.*]] = add i64 [[LD1]], [[LD2]]
; NO-UNIT-STRIDE-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; NO-UNIT-STRIDE-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 64
; NO-UNIT-STRIDE-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
; NO-UNIT-STRIDE-MV: [[EXIT]]:
; NO-UNIT-STRIDE-MV-NEXT: ret void
;
entry:
%add = add nsw nuw i64 %stride, 2
%mul = mul nsw nuw i64 %add, %add
%out.offset = add i64 %mul, 16
%out = getelementptr i64, ptr %p, i64 %out.offset
br label %header
header:
%iv = phi i64 [ 1, %entry ], [ %iv.next, %header ]
%iv.next = add nsw i64 %iv, 1
%idx = mul i64 %iv, %stride
%gep.ld = getelementptr i64, ptr %p, i64 %iv
%gep.st = getelementptr i64, ptr %out, i64 %iv
%gep.strided = getelementptr i64, ptr %p.strided, i64 %idx
%ld1 = load i64, ptr %gep.ld
%ld2 = load i64, ptr %gep.strided
%val = add i64 %ld1, %ld2
store i64 %val, ptr %gep.st
%exitcond = icmp slt i64 %iv.next, 64
br i1 %exitcond, label %header, label %exit
exit:
ret void
}
Reference in New Issue View Git Blame Copy Permalink