shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[VPlan] Introduce VPScalarPHIRecipe, use for can & EVL IV codegen (NFC). (#114305)

Browse Source 
Introduce a general recipe to generate a scalar phi. Lower
VPCanonicalIVPHIRecipe and VPEVLBasedIVRecipe to VPScalarIVPHIrecipe
before plan execution, avoiding the need for duplicated ::execute
implementations. There are other cases that could benefit, including
in-loop reduction phis and pointer induction phis.

Builds on a similar idea as
https://github.com/llvm/llvm-project/pull/82270.

PR: https://github.com/llvm/llvm-project/pull/114305
This commit is contained in:
Florian Hahn 2024-12-03 14:53:51 +00:00 committed by GitHub
parent c7e14689dd
commit a7fda0e1e4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 109 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -7721,6 +7721,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan.prepareToExecute(ILV.getTripCount(),
ILV.getOrCreateVectorTripCount(nullptr),
CanonicalIVStartValue, State);
VPlanTransforms::prepareToExecute(BestVPlan);
BestVPlan.execute(&State);

7
llvm/lib/Transforms/Vectorize/VPlan.cpp
View File

@ -1070,10 +1070,9 @@ void VPlan::execute(VPTransformState *State) {
}
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
bool NeedsScalar =
isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(PhiR) ||
(isa<VPReductionPHIRecipe>(PhiR) &&
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
bool NeedsScalar = isa<VPScalarPHIRecipe>(PhiR) ||
(isa<VPReductionPHIRecipe>(PhiR) &&
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
Value *Phi = State->get(PhiR, NeedsScalar);
Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);

52
llvm/lib/Transforms/Vectorize/VPlan.h
View File

@ -2239,6 +2239,45 @@ public:
#endif
};
/// Recipe to generate a scalar PHI. Used to generate code for recipes that
/// produce scalar header phis, including VPCanonicalIVPHIRecipe and
/// VPEVLBasedIVPHIRecipe.
class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
std::string Name;
public:
VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
StringRef Name)
: VPHeaderPHIRecipe(VPDef::VPScalarPHISC, nullptr, Start, DL),
Name(Name.str()) {
addOperand(BackedgeValue);
}
~VPScalarPHIRecipe() override = default;
VPScalarPHIRecipe *clone() override {
llvm_unreachable("cloning not implemented yet");
}
VP_CLASSOF_IMPL(VPDef::VPScalarPHISC)
/// Generate the phi/select nodes.
void execute(VPTransformState &State) override;
/// Returns true if the recipe only uses the first lane of operand \p Op.
bool onlyFirstLaneUsed(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
"Op must be an operand of the recipe");
return true;
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
};
/// A recipe for handling phis that are widened in the vector loop.
/// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
/// managed in the recipe directly.
@ -3134,8 +3173,10 @@ public:
return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
}
/// Generate the canonical scalar induction phi of the vector loop.
void execute(VPTransformState &State) override;
void execute(VPTransformState &State) override {
llvm_unreachable(
"cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@ -3231,9 +3272,10 @@ public:
return D->getVPDefID() == VPDef::VPEVLBasedIVPHISC;
}
/// Generate phi for handling IV based on EVL over iterations correctly.
/// TODO: investigate if it can share the code with VPCanonicalIVPHIRecipe.
void execute(VPTransformState &State) override;
void execute(VPTransformState &State) override {
llvm_unreachable(
"cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
}
/// Return the cost of this VPEVLBasedIVPHIRecipe.
InstructionCost computeCost(ElementCount VF,

16
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
View File

@ -213,14 +213,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
[this](const auto *R) {
// Handle header phi recipes, except VPWidenIntOrFpInduction
// which needs special handling due it being possibly truncated.
// TODO: consider inferring/caching type of siblings, e.g.,
// backedge value, here and in cases below.
return inferScalarType(R->getStartValue());
})
VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe,
VPScalarPHIRecipe>([this](const auto *R) {
// Handle header phi recipes, except VPWidenIntOrFpInduction
// which needs special handling due it being possibly truncated.
// TODO: consider inferring/caching type of siblings, e.g.,
// backedge value, here and in cases below.
return inferScalarType(R->getStartValue());
})
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
[](const auto *R) { return R->getScalarType(); })
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,

47
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
View File

@ -3101,17 +3101,6 @@ InstructionCost VPInterleaveRecipe::computeCost(ElementCount VF,
VectorTy, std::nullopt, CostKind, 0);
}
void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) {
Value *Start = getStartValue()->getLiveInIRValue();
PHINode *Phi = PHINode::Create(Start->getType(), 2, "index");
Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
Phi->addIncoming(Start, VectorPH);
Phi->setDebugLoc(getDebugLoc());
State.set(this, Phi, /*IsScalar*/ true);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPCanonicalIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
@ -3153,8 +3142,6 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
assert(!onlyScalarsGenerated(State.VF.isScalable()) &&
"Recipe should have been replaced");
auto *IVR = getParent()->getPlan()->getCanonicalIV();
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
unsigned CurrentPart = getUnrollPart(*this);
// Build a pointer phi
@ -3164,6 +3151,12 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
PHINode *NewPointerPhi = nullptr;
if (CurrentPart == 0) {
auto *IVR = cast<VPHeaderPHIRecipe>(&getParent()
->getPlan()
->getVectorLoopRegion()
->getEntryBasicBlock()
->front());
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",
CanonicalIV->getIterator());
NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
@ -3477,15 +3470,6 @@ void VPActiveLaneMaskPHIRecipe::print(raw_ostream &O, const Twine &Indent,
}
#endif
void VPEVLBasedIVPHIRecipe::execute(VPTransformState &State) {
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
Value *Start = State.get(getOperand(0), VPLane(0));
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, "evl.based.iv");
Phi->addIncoming(Start, VectorPH);
Phi->setDebugLoc(getDebugLoc());
State.set(this, Phi, /*IsScalar=*/true);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPEVLBasedIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
@ -3496,3 +3480,22 @@ void VPEVLBasedIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
printOperands(O, SlotTracker);
}
#endif
void VPScalarPHIRecipe::execute(VPTransformState &State) {
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
Value *Start = State.get(getOperand(0), VPLane(0));
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
Phi->addIncoming(Start, VectorPH);
Phi->setDebugLoc(getDebugLoc());
State.set(this, Phi, /*IsScalar=*/true);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPScalarPHIRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "SCALAR-PHI";
printAsOperand(O, SlotTracker);
O << " = phi ";
printOperands(O, SlotTracker);
}
#endif

21
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View File

@ -1819,3 +1819,24 @@ void VPlanTransforms::createInterleaveGroups(
}
}
}
void VPlanTransforms::prepareToExecute(VPlan &Plan) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getVectorLoopRegion());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(VPBB->phis())) {
if (!isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(&R))
continue;
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
StringRef Name =
isa<VPCanonicalIVPHIRecipe>(PhiR) ? "index" : "evl.based.iv";
auto *ScalarR =
new VPScalarPHIRecipe(PhiR->getStartValue(), PhiR->getBackedgeValue(),
PhiR->getDebugLoc(), Name);
ScalarR->insertBefore(PhiR);
PhiR->replaceAllUsesWith(ScalarR);
PhiR->eraseFromParent();
}
}
}

3
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
View File

@ -123,6 +123,9 @@ struct VPlanTransforms {
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
/// Lower abstract recipes to concrete ones, that can be codegen'd.
static void prepareToExecute(VPlan &Plan);
};
} // namespace llvm

1
llvm/lib/Transforms/Vectorize/VPlanValue.h
View File

@ -359,6 +359,7 @@ public:
VPFirstOrderRecurrencePHISC,
VPWidenIntOrFpInductionSC,
VPWidenPointerInductionSC,
VPScalarPHISC,
VPReductionPHISC,
// END: SubclassID for recipes that inherit VPHeaderPHIRecipe
// END: Phi-like recipes