shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[VPlan] Replace VPBB for vector.ph during skeleton creation (NFC)

Browse Source 
Shift replacement of regular VPBB for vector.ph with the VPIRBB wrapping
the created IR block directly to skeleton creation, to be consistent
with how the scalar preheader is handled.
This commit is contained in:
Florian Hahn 2025-08-13 08:30:17 +01:00
parent dfe18b1a0e
commit 48bfaa4c06
No known key found for this signature in database
GPG Key ID: C8B0D7090F9127E6
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -511,7 +511,8 @@ public:
MinProfitableTripCount(MinProfitableTripCount), UF(UnrollFactor),
Builder(PSE.getSE()->getContext()), Cost(CM), BFI(BFI), PSI(PSI),
RTChecks(RTChecks), Plan(Plan),
VectorPHVPB(Plan.getVectorLoopRegion()->getSinglePredecessor()) {}
VectorPHVPBB(cast<VPBasicBlock>(
Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
virtual ~InnerLoopVectorizer() = default;
@ -645,7 +646,7 @@ protected:
/// The vector preheader block of \p Plan, used as target for check blocks
/// introduced during skeleton creation.
VPBlockBase *VectorPHVPB;
VPBasicBlock *VectorPHVPBB;
};
/// Encapsulate information regarding vectorization of a loop and its epilogue.
@ -2277,11 +2278,11 @@ void InnerLoopVectorizer::introduceCheckBlockInVPlan(BasicBlock *CheckIRBB) {
// Note: The block with the minimum trip-count check is already connected
// during earlier VPlan construction.
VPBlockBase *ScalarPH = Plan.getScalarPreheader();
VPBlockBase *PreVectorPH = VectorPHVPB->getSinglePredecessor();
VPBlockBase *PreVectorPH = VectorPHVPBB->getSinglePredecessor();
assert(PreVectorPH->getNumSuccessors() == 2 && "Expected 2 successors");
assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor");
VPIRBasicBlock *CheckVPIRBB = Plan.createVPIRBasicBlock(CheckIRBB);
VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPB, CheckVPIRBB);
VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPBB, CheckVPIRBB);
PreVectorPH = CheckVPIRBB;
VPBlockUtils::connectBlocks(PreVectorPH, ScalarPH);
PreVectorPH->swapSuccessors();
@ -2388,7 +2389,8 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
/// VPBB are moved to the end of the newly created VPIRBasicBlock. VPBB must
/// have a single predecessor, which is rewired to the new VPIRBasicBlock. All
/// successors of VPBB, if any, are rewired to the new VPIRBasicBlock.
static void replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB) {
static VPIRBasicBlock *replaceVPBBWithIRVPBB(VPBasicBlock *VPBB,
BasicBlock *IRBB) {
VPIRBasicBlock *IRVPBB = VPBB->getPlan()->createVPIRBasicBlock(IRBB);
auto IP = IRVPBB->begin();
for (auto &R : make_early_inc_range(VPBB->phis()))
@ -2400,6 +2402,7 @@ static void replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB) {
VPBlockUtils::reassociateBlocks(VPBB, IRVPBB);
// VPBB is now dead and will be cleaned up when the plan gets destroyed.
return IRVPBB;
}
void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
@ -2503,6 +2506,7 @@ BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
emitIterationCountCheck(LoopScalarPreHeader);
replaceVPBBWithIRVPBB(Plan.getScalarPreheader(), LoopScalarPreHeader);
replaceVPBBWithIRVPBB(VectorPHVPBB, LoopVectorPreHeader);
return LoopVectorPreHeader;
}
@ -7363,9 +7367,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
//
//===------------------------------------------------===//
// 2. Copy and widen instructions from the old loop into the new loop.
replaceVPBBWithIRVPBB(VectorPH, State.CFG.PrevBB);
// Move check blocks to their final position.
// TODO: Move as part of VPIRBB execute and update impacted tests.
if (BasicBlock *MemCheckBlock = ILV.RTChecks.getMemRuntimeChecks().second)
@ -7521,12 +7522,13 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
LoopVectorPreHeader = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
static_cast<DominatorTree *>(nullptr), LI,
nullptr, "vector.ph");
if (ForEpilogue) {
// Save the trip count so we don't have to regenerate it in the
// vec.epilog.iter.check. This is safe to do because the trip count
// generated here dominates the vector epilog iter check.
EPI.TripCount = Count;
} else {
VectorPHVPBB = replaceVPBBWithIRVPBB(VectorPHVPBB, LoopVectorPreHeader);
}
BranchInst &BI =
@ -7560,6 +7562,8 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
BasicBlock *VecEpilogueIterationCountCheck =
SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->begin(), DT, LI,
nullptr, "vec.epilog.iter.check", true);
VectorPHVPBB = replaceVPBBWithIRVPBB(VectorPHVPBB, LoopVectorPreHeader);
emitMinimumVectorEpilogueIterCountCheck(LoopScalarPreHeader,
VecEpilogueIterationCountCheck);
AdditionalBypassBlock = VecEpilogueIterationCountCheck;