From 8d442bc5b58c9c8e2a5c321a06c8d9017630ca44 Mon Sep 17 00:00:00 2001 From: vporpo Date: Mon, 6 Apr 2026 11:25:20 -0700 Subject: [PATCH] [SandboxVec][LoadStoreVec] Add support for constants (#189769) Up until now the pass would only vectorize load-store pairs. This patch implements vectorization of constant-store pairs. --- .../SandboxVectorizer/Passes/LoadStoreVec.h | 2 +- .../SandboxVectorizer/Passes/LoadStoreVec.cpp | 108 ++++++++++++------ .../SandboxVectorizer/load_store_vec.ll | 54 +++++++++ 3 files changed, 125 insertions(+), 39 deletions(-) diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h index eeb735bda34c..1f225a96313b 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h +++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h @@ -36,7 +36,7 @@ class LoadStoreVec final : public RegionPass { Scheduler &Sched); void tryEraseDeadInstrs(ArrayRef Stores, - ArrayRef Loads); + ArrayRef Operands); public: LoadStoreVec() : RegionPass("load-store-vec") {} diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp index 827debb889b1..0436f963597a 100644 --- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp @@ -48,7 +48,7 @@ std::optional LoadStoreVec::canVectorize(ArrayRef Bndl, } void LoadStoreVec::tryEraseDeadInstrs(ArrayRef Stores, - ArrayRef Loads) { + ArrayRef Operands) { SmallPtrSet DeadCandidates; for (auto *SI : Stores) { if (auto *PtrI = @@ -56,11 +56,13 @@ void LoadStoreVec::tryEraseDeadInstrs(ArrayRef Stores, DeadCandidates.insert(PtrI); SI->eraseFromParent(); } - for (auto *LI : Loads) { - if (auto *PtrI = - dyn_cast(cast(LI)->getPointerOperand())) - DeadCandidates.insert(PtrI); - cast(LI)->eraseFromParent(); + for (auto *Op : Operands) { + if (auto *LI = dyn_cast(Op)) { + if (auto *PtrI = + dyn_cast(cast(LI)->getPointerOperand())) + DeadCandidates.insert(PtrI); + cast(LI)->eraseFromParent(); + } } for (auto *PtrI : DeadCandidates) if (!PtrI->hasNUsesOrMore(1)) @@ -91,46 +93,76 @@ bool LoadStoreVec::runOnRegion(Region &Rgn, const Analyses &A) { // TODO: For now we only support load operands. // TODO: For now we don't cross BBs. // TODO: For now don't vectorize if the loads have external uses. - if (!all_of(Operands, [BB](Value *V) { - auto *LI = dyn_cast(V); - if (LI == nullptr) - return false; - if (LI->getParent() != BB) - return false; - if (LI->hasNUsesOrMore(2)) - return false; - return true; - })) - return false; - // TODO: Try to avoid the extra copy to an instruction vector. - SmallVector Loads; - Loads.reserve(Operands.size()); - for (Value *Op : Operands) - Loads.push_back(cast(Op)); - - bool Consecutive = VecUtils::areConsecutive( - Loads, A.getScalarEvolution(), *DL); - if (!Consecutive) - return false; - if (!canVectorize(Loads, Sched)) + bool AllLoads = all_of(Operands, [BB](Value *V) { + auto *LI = dyn_cast(V); + if (LI == nullptr) + return false; + // TODO: For now we don't cross BBs. + if (LI->getParent() != BB) + return false; + if (LI->hasNUsesOrMore(2)) + return false; + return true; + }); + bool AllConstants = + all_of(Operands, [](Value *V) { return isa(V); }); + if (!AllLoads && !AllConstants) return false; - // Generate vector store and vector load - Type *Ty = VecUtils::getCombinedVectorTypeFor(Bndl, *DL); - Value *LdPtr = cast(Loads[0])->getPointerOperand(); - // TODO: Compute alignment. - Align LdAlign(1); - auto LdWhereIt = std::next(VecUtils::getLowest(Loads)->getIterator()); - auto *VecLd = - LoadInst::create(Ty, LdPtr, LdAlign, LdWhereIt, Ctx, "VecIinitL"); + Value *VecOp = nullptr; + if (AllLoads) { + // TODO: Try to avoid the extra copy to an instruction vector. + SmallVector Loads; + Loads.reserve(Operands.size()); + for (Value *Op : Operands) + Loads.push_back(cast(Op)); + bool Consecutive = VecUtils::areConsecutive( + Loads, A.getScalarEvolution(), *DL); + if (!Consecutive) + return false; + if (!canVectorize(Loads, Sched)) + return false; + + // Generate vector load. + Type *Ty = VecUtils::getCombinedVectorTypeFor(Bndl, *DL); + Value *LdPtr = cast(Loads[0])->getPointerOperand(); + // TODO: Compute alignment. + Align LdAlign(1); + auto LdWhereIt = std::next(VecUtils::getLowest(Loads)->getIterator()); + VecOp = LoadInst::create(Ty, LdPtr, LdAlign, LdWhereIt, Ctx, "VecIinitL"); + } else if (AllConstants) { + SmallVector Constants; + Constants.reserve(Operands.size()); + for (Value *Op : Operands) { + auto *COp = cast(Op); + if (auto *AggrCOp = dyn_cast(COp)) { + // If the operand is a constant aggregate, then append all its elements. + for (Value *Elm : AggrCOp->operands()) + Constants.push_back(cast(Elm)); + } else if (auto *SeqCOp = dyn_cast(COp)) { + for (auto ElmIdx : seq(SeqCOp->getNumElements())) + Constants.push_back(SeqCOp->getElementAsConstant(ElmIdx)); + } else if (auto *Zero = dyn_cast(COp)) { + auto *ZeroElm = Zero->getSequentialElement(); + for ([[maybe_unused]] auto Cnt : + seq(Zero->getElementCount().getFixedValue())) + Constants.push_back(ZeroElm); + } else { + Constants.push_back(COp); + } + } + VecOp = ConstantVector::get(Constants); + } + + // Generate vector store. Value *StPtr = cast(Bndl[0])->getPointerOperand(); // TODO: Compute alignment. Align StAlign(1); auto StWhereIt = std::next(VecUtils::getLowest(Bndl)->getIterator()); - StoreInst::create(VecLd, StPtr, StAlign, StWhereIt, Ctx); + StoreInst::create(VecOp, StPtr, StAlign, StWhereIt, Ctx); - tryEraseDeadInstrs(Bndl, Loads); + tryEraseDeadInstrs(Bndl, Operands); return true; } diff --git a/llvm/test/Transforms/SandboxVectorizer/load_store_vec.ll b/llvm/test/Transforms/SandboxVectorizer/load_store_vec.ll index 972a5a7a73a9..76d46bd70943 100644 --- a/llvm/test/Transforms/SandboxVectorizer/load_store_vec.ll +++ b/llvm/test/Transforms/SandboxVectorizer/load_store_vec.ll @@ -301,6 +301,57 @@ define void @load_store_vec_external_uses(ptr %ptr) { ret void } +; Store-constant pattern. +define void @load_store_vec_constants(ptr %ptr) { +; CHECK-LABEL: define void @load_store_vec_constants( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0 +; CHECK-NEXT: store <3 x i8> , ptr [[PTR0]], align 1, !sandboxvec [[META15:![0-9]+]] +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr i8, ptr %ptr, i32 0 + %ptr1 = getelementptr i8, ptr %ptr, i32 1 + %ptr2 = getelementptr i8, ptr %ptr, i32 3 + store i8 42, ptr %ptr0 + store i16 43, ptr %ptr1 + store i8 44, ptr %ptr2 + ret void +} + +; Same but with ConstantDataSequential. +define void @load_store_vec_constants_CDS(ptr %ptr) { +; CHECK-LABEL: define void @load_store_vec_constants_CDS( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0 +; CHECK-NEXT: store <4 x i8> , ptr [[PTR0]], align 1, !sandboxvec [[META16:![0-9]+]] +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr i8, ptr %ptr, i32 0 + %ptr1 = getelementptr i8, ptr %ptr, i32 1 + %ptr2 = getelementptr i8, ptr %ptr, i32 3 + store i8 0, ptr %ptr0 + store <2 x i8> , ptr %ptr1 + store i8 3, ptr %ptr2 + ret void +} + +; Same but with floats +define void @load_store_vec_constants_CDS_float(ptr %ptr) { +; CHECK-LABEL: define void @load_store_vec_constants_CDS_float( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[PTR0:%.*]] = getelementptr float, ptr [[PTR]], i32 0 +; CHECK-NEXT: store <8 x float> , ptr [[PTR0]], align 1, !sandboxvec [[META17:![0-9]+]] +; CHECK-NEXT: ret void +; + %ptr0 = getelementptr float, ptr %ptr, i32 0 + %ptr1 = getelementptr float, ptr %ptr, i32 1 + %ptr2 = getelementptr float, ptr %ptr, i32 3 + store float 1.0, ptr %ptr0 + store <2 x float> zeroinitializer, ptr %ptr1 + store <5 x float> zeroinitializer, ptr %ptr2 + ret void +} + ;. ; CHECK: [[META0]] = distinct !{!"sandboxregion"} ; CHECK: [[META1]] = distinct !{!"sandboxregion"} @@ -317,4 +368,7 @@ define void @load_store_vec_external_uses(ptr %ptr) { ; CHECK: [[META12]] = distinct !{!"sandboxregion"} ; CHECK: [[META13]] = distinct !{!"sandboxregion"} ; CHECK: [[META14]] = distinct !{!"sandboxregion"} +; CHECK: [[META15]] = distinct !{!"sandboxregion"} +; CHECK: [[META16]] = distinct !{!"sandboxregion"} +; CHECK: [[META17]] = distinct !{!"sandboxregion"} ;.