shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/4x2xhalf.ll
Drew Kersnar 5c8c7f3d21
[LoadStoreVectorizer] Fill gaps in load/store chains to enable vectorization (#159388) 
This change introduces Gap Filling, an optimization that aims to fill in
holes in otherwise contiguous load/store chains to enable vectorization.
It also introduces Chain Extending, which extends the end of a chain to
the closest power of 2.

This was originally motivated by the NVPTX target, but I tried to
generalize it to be universally applicable to all targets that may use
the LSV. I'm more than willing to make adjustments to improve the
target-agnostic-ness of this change. I fully expect there are some
issues and encourage feedback on how to improve things.

For both loads and stores we only perform the optimization when we can
generate a legal llvm masked load/store intrinsic, masking off the
"extra" elements. Determining legality for stores is a little tricky
from the NVPTX side, because these intrinsics are only supported for
256-bit vectors. See the other PR I opened for the implementation of the
NVPTX lowering of masked store intrinsics, which include NVPTX TTI
changes that return true for isLegalMaskedStore under certain
conditions: https://github.com/llvm/llvm-project/pull/159387. This
change is dependent on that backend change, but I predict this change
will require more discussion, so I am putting them both up at the same
time. The backend change will be merged first assuming both are
approved.

Edited: both stores _and loads_ must use masked intrinsics for this
optimization to be legal.

---------

Co-authored-by: Matt Arsenault <arsenm2@gmail.com>
2025-12-08 15:57:17 -06:00

156 lines
9.5 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
define void @ldg_f16(ptr nocapture align 16 %rd0) {
; CHECK-LABEL: define void @ldg_f16(
; CHECK-SAME: ptr align 16 captures(none) [[RD0:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x half>, ptr [[RD0]], align 16
; CHECK-NEXT: [[LOAD11:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[LOAD22:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: [[LOAD33:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: [[LOAD44:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <2 x i32> <i32 6, i32 7>
; CHECK-NEXT: [[P1:%.*]] = fcmp ogt <2 x half> [[LOAD11]], zeroinitializer
; CHECK-NEXT: [[S1:%.*]] = select <2 x i1> [[P1]], <2 x half> [[LOAD11]], <2 x half> zeroinitializer
; CHECK-NEXT: [[P2:%.*]] = fcmp ogt <2 x half> [[LOAD22]], zeroinitializer
; CHECK-NEXT: [[S2:%.*]] = select <2 x i1> [[P2]], <2 x half> [[LOAD22]], <2 x half> zeroinitializer
; CHECK-NEXT: [[P3:%.*]] = fcmp ogt <2 x half> [[LOAD33]], zeroinitializer
; CHECK-NEXT: [[S3:%.*]] = select <2 x i1> [[P3]], <2 x half> [[LOAD33]], <2 x half> zeroinitializer
; CHECK-NEXT: [[P4:%.*]] = fcmp ogt <2 x half> [[LOAD44]], zeroinitializer
; CHECK-NEXT: [[S4:%.*]] = select <2 x i1> [[P4]], <2 x half> [[LOAD44]], <2 x half> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x half> [[S1]], i32 0
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x half> poison, half [[TMP2]], i32 0
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x half> [[S1]], i32 1
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x half> [[TMP3]], half [[TMP4]], i32 1
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x half> [[S2]], i32 0
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x half> [[TMP5]], half [[TMP6]], i32 2
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x half> [[S2]], i32 1
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x half> [[TMP7]], half [[TMP8]], i32 3
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x half> [[S3]], i32 0
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x half> [[TMP9]], half [[TMP10]], i32 4
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x half> [[S3]], i32 1
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x half> [[TMP11]], half [[TMP12]], i32 5
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x half> [[S4]], i32 0
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x half> [[TMP13]], half [[TMP14]], i32 6
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x half> [[S4]], i32 1
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x half> [[TMP15]], half [[TMP16]], i32 7
; CHECK-NEXT: store <8 x half> [[TMP17]], ptr [[RD0]], align 16
; CHECK-NEXT: ret void
;
%load1 = load <2 x half>, ptr %rd0, align 16
%p1 = fcmp ogt <2 x half> %load1, zeroinitializer
%s1 = select <2 x i1> %p1, <2 x half> %load1, <2 x half> zeroinitializer
store <2 x half> %s1, ptr %rd0, align 16
%in2 = getelementptr half, ptr %rd0, i64 2
%load2 = load <2 x half>, ptr %in2, align 4
%p2 = fcmp ogt <2 x half> %load2, zeroinitializer
%s2 = select <2 x i1> %p2, <2 x half> %load2, <2 x half> zeroinitializer
store <2 x half> %s2, ptr %in2, align 4
%in3 = getelementptr half, ptr %rd0, i64 4
%load3 = load <2 x half>, ptr %in3, align 4
%p3 = fcmp ogt <2 x half> %load3, zeroinitializer
%s3 = select <2 x i1> %p3, <2 x half> %load3, <2 x half> zeroinitializer
store <2 x half> %s3, ptr %in3, align 4
%in4 = getelementptr half, ptr %rd0, i64 6
%load4 = load <2 x half>, ptr %in4, align 4
%p4 = fcmp ogt <2 x half> %load4, zeroinitializer
%s4 = select <2 x i1> %p4, <2 x half> %load4, <2 x half> zeroinitializer
store <2 x half> %s4, ptr %in4, align 4
ret void
}
define void @no_nonpow2_vector(ptr nocapture align 16 %rd0) {
; CHECK-LABEL: define void @no_nonpow2_vector(
; CHECK-SAME: ptr align 16 captures(none) [[RD0:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = call <8 x half> @llvm.masked.load.v8f16.p0(ptr align 16 [[RD0]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false>, <8 x half> poison)
; CHECK-NEXT: [[LOAD13:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[LOAD24:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[EXTEND5:%.*]] = extractelement <8 x half> [[TMP1]], i32 6
; CHECK-NEXT: [[EXTEND26:%.*]] = extractelement <8 x half> [[TMP1]], i32 7
; CHECK-NEXT: [[P1:%.*]] = fcmp ogt <3 x half> [[LOAD13]], zeroinitializer
; CHECK-NEXT: [[S1:%.*]] = select <3 x i1> [[P1]], <3 x half> [[LOAD13]], <3 x half> zeroinitializer
; CHECK-NEXT: store <3 x half> [[S1]], ptr [[RD0]], align 16
; CHECK-NEXT: [[IN2:%.*]] = getelementptr half, ptr [[RD0]], i64 3
; CHECK-NEXT: [[P2:%.*]] = fcmp ogt <3 x half> [[LOAD24]], zeroinitializer
; CHECK-NEXT: [[S2:%.*]] = select <3 x i1> [[P2]], <3 x half> [[LOAD24]], <3 x half> zeroinitializer
; CHECK-NEXT: store <3 x half> [[S2]], ptr [[IN2]], align 4
; CHECK-NEXT: [[IN3:%.*]] = getelementptr half, ptr [[RD0]], i64 6
; CHECK-NEXT: [[LOAD3:%.*]] = load <3 x half>, ptr [[IN3]], align 4
; CHECK-NEXT: [[P3:%.*]] = fcmp ogt <3 x half> [[LOAD3]], zeroinitializer
; CHECK-NEXT: [[S3:%.*]] = select <3 x i1> [[P3]], <3 x half> [[LOAD3]], <3 x half> zeroinitializer
; CHECK-NEXT: store <3 x half> [[S3]], ptr [[IN3]], align 4
; CHECK-NEXT: [[IN4:%.*]] = getelementptr half, ptr [[RD0]], i64 9
; CHECK-NEXT: [[LOAD4:%.*]] = load <3 x half>, ptr [[IN4]], align 4
; CHECK-NEXT: [[P4:%.*]] = fcmp ogt <3 x half> [[LOAD4]], zeroinitializer
; CHECK-NEXT: [[S4:%.*]] = select <3 x i1> [[P4]], <3 x half> [[LOAD4]], <3 x half> zeroinitializer
; CHECK-NEXT: store <3 x half> [[S4]], ptr [[IN4]], align 4
; CHECK-NEXT: ret void
;
%load1 = load <3 x half>, ptr %rd0, align 16
%p1 = fcmp ogt <3 x half> %load1, zeroinitializer
%s1 = select <3 x i1> %p1, <3 x half> %load1, <3 x half> zeroinitializer
store <3 x half> %s1, ptr %rd0, align 16
%in2 = getelementptr half, ptr %rd0, i64 3
%load2 = load <3 x half>, ptr %in2, align 4
%p2 = fcmp ogt <3 x half> %load2, zeroinitializer
%s2 = select <3 x i1> %p2, <3 x half> %load2, <3 x half> zeroinitializer
store <3 x half> %s2, ptr %in2, align 4
%in3 = getelementptr half, ptr %rd0, i64 6
%load3 = load <3 x half>, ptr %in3, align 4
%p3 = fcmp ogt <3 x half> %load3, zeroinitializer
%s3 = select <3 x i1> %p3, <3 x half> %load3, <3 x half> zeroinitializer
store <3 x half> %s3, ptr %in3, align 4
%in4 = getelementptr half, ptr %rd0, i64 9
%load4 = load <3 x half>, ptr %in4, align 4
%p4 = fcmp ogt <3 x half> %load4, zeroinitializer
%s4 = select <3 x i1> %p4, <3 x half> %load4, <3 x half> zeroinitializer
store <3 x half> %s4, ptr %in4, align 4
ret void
}
define void @no_pointer_vector(ptr nocapture align 16 %rd0) {
; CHECK-LABEL: define void @no_pointer_vector(
; CHECK-SAME: ptr align 16 captures(none) [[RD0:%.*]]) {
; CHECK-NEXT: [[LOAD1:%.*]] = load <2 x ptr>, ptr [[RD0]], align 16
; CHECK-NEXT: [[P1:%.*]] = icmp ne <2 x ptr> [[LOAD1]], zeroinitializer
; CHECK-NEXT: [[S1:%.*]] = select <2 x i1> [[P1]], <2 x ptr> [[LOAD1]], <2 x ptr> zeroinitializer
; CHECK-NEXT: store <2 x ptr> [[S1]], ptr [[RD0]], align 16
; CHECK-NEXT: [[IN2:%.*]] = getelementptr ptr, ptr [[RD0]], i64 2
; CHECK-NEXT: [[LOAD2:%.*]] = load <2 x ptr>, ptr [[IN2]], align 4
; CHECK-NEXT: [[P2:%.*]] = icmp ne <2 x ptr> [[LOAD2]], zeroinitializer
; CHECK-NEXT: [[S2:%.*]] = select <2 x i1> [[P2]], <2 x ptr> [[LOAD2]], <2 x ptr> zeroinitializer
; CHECK-NEXT: store <2 x ptr> [[S2]], ptr [[IN2]], align 4
; CHECK-NEXT: [[IN3:%.*]] = getelementptr ptr, ptr [[RD0]], i64 4
; CHECK-NEXT: [[LOAD3:%.*]] = load <2 x ptr>, ptr [[IN3]], align 4
; CHECK-NEXT: [[P3:%.*]] = icmp ne <2 x ptr> [[LOAD3]], zeroinitializer
; CHECK-NEXT: [[S3:%.*]] = select <2 x i1> [[P3]], <2 x ptr> [[LOAD3]], <2 x ptr> zeroinitializer
; CHECK-NEXT: store <2 x ptr> [[S3]], ptr [[IN3]], align 4
; CHECK-NEXT: [[IN4:%.*]] = getelementptr ptr, ptr [[RD0]], i64 6
; CHECK-NEXT: [[LOAD4:%.*]] = load <2 x ptr>, ptr [[IN4]], align 4
; CHECK-NEXT: [[P4:%.*]] = icmp ne <2 x ptr> [[LOAD4]], zeroinitializer
; CHECK-NEXT: [[S4:%.*]] = select <2 x i1> [[P4]], <2 x ptr> [[LOAD4]], <2 x ptr> zeroinitializer
; CHECK-NEXT: store <2 x ptr> [[S4]], ptr [[IN4]], align 4
; CHECK-NEXT: ret void
;
%load1 = load <2 x ptr>, ptr %rd0, align 16
%p1 = icmp ne <2 x ptr> %load1, zeroinitializer
%s1 = select <2 x i1> %p1, <2 x ptr> %load1, <2 x ptr> zeroinitializer
store <2 x ptr> %s1, ptr %rd0, align 16
%in2 = getelementptr ptr, ptr %rd0, i64 2
%load2 = load <2 x ptr>, ptr %in2, align 4
%p2 = icmp ne <2 x ptr> %load2, zeroinitializer
%s2 = select <2 x i1> %p2, <2 x ptr> %load2, <2 x ptr> zeroinitializer
store <2 x ptr> %s2, ptr %in2, align 4
%in3 = getelementptr ptr, ptr %rd0, i64 4
%load3 = load <2 x ptr>, ptr %in3, align 4
%p3 = icmp ne <2 x ptr> %load3, zeroinitializer
%s3 = select <2 x i1> %p3, <2 x ptr> %load3, <2 x ptr> zeroinitializer
store <2 x ptr> %s3, ptr %in3, align 4
%in4 = getelementptr ptr, ptr %rd0, i64 6
%load4 = load <2 x ptr>, ptr %in4, align 4
%p4 = icmp ne <2 x ptr> %load4, zeroinitializer
%s4 = select <2 x i1> %p4, <2 x ptr> %load4, <2 x ptr> zeroinitializer
store <2 x ptr> %s4, ptr %in4, align 4
ret void
}
Reference in New Issue View Git Blame Copy Permalink