shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[VPlan] Populate and use VPIRFlags from initial VPInstruction. (#168450)

Browse Source 
Update VPlan to populate VPIRFlags during VPInstruction construction and
use it when creating widened recipes, instead of constructing VPIRFlags
from the underlying IR instruction each time. The VPRecipeWithIRFlags
constructor taking an underlying instruction and setting the flags based
on it has been removed.

This centralizes initial VPIRFlags creation and ensures flags are
consistently available throughout VPlan transformations and makes sure
we don't accidentally re-add flags from the underlying instruction that
already got dropped during transformations.

Follow-up to https://github.com/llvm/llvm-project/pull/167253, which did
the same for VPIRMetadata.

Should be NFC w.r.t. to the generated IR.

PR: https://github.com/llvm/llvm-project/pull/168450
This commit is contained in:
Florian Hahn 2025-11-18 15:15:14 +00:00 committed by GitHub
parent 9a0fd22da1
commit 2befda2225
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 134 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
View File

@ -152,11 +152,12 @@ public:
/// its underlying Instruction.
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
Instruction *Inst = nullptr,
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
VPInstruction *NewVPInst = tryInsertInstruction(
new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
new VPInstruction(Opcode, Operands, Flags, MD, DL, Name));
NewVPInst->setUnderlyingValue(Inst);
return NewVPInst;
}
@ -329,7 +330,7 @@ public:
else if (Opcode == Instruction::ZExt)
Flags = VPIRFlags::NonNegFlagsTy(false);
return tryInsertInstruction(
new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags));
new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags));
}
VPScalarIVStepsRecipe *

35
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -7750,7 +7750,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
},
Range);
if (ShouldUseVectorIntrinsic)
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI,
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI, *VPI,
VPI->getDebugLoc());
Function *Variant = nullptr;
@ -7804,7 +7804,8 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
}
Ops.push_back(VPI->getOperand(VPI->getNumOperands() - 1));
return new VPWidenCallRecipe(CI, Variant, Ops, VPI->getDebugLoc());
return new VPWidenCallRecipe(CI, Variant, Ops, *VPI, *VPI,
VPI->getDebugLoc());
}
return nullptr;
@ -7842,7 +7843,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
auto *SafeRHS =
Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
Ops[1] = SafeRHS;
return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc());
return new VPWidenRecipe(*I, Ops, *VPI, *VPI, VPI->getDebugLoc());
}
[[fallthrough]];
}
@ -7888,7 +7889,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
// For other binops, the legacy cost model only checks the second operand.
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
}
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(VPI->operands());
@ -7896,7 +7897,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
NewOps.push_back(Plan.getConstantInt(32, Idx));
return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
}
};
}
@ -7981,7 +7982,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
auto *Recipe =
new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI);
new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI,
*VPI, VPI->getDebugLoc());
return Recipe;
}
@ -8231,17 +8233,19 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
return nullptr;
if (VPI->getOpcode() == Instruction::GetElementPtr)
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands());
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands(),
*VPI, VPI->getDebugLoc());
if (VPI->getOpcode() == Instruction::Select)
return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands(),
*VPI);
return new VPWidenSelectRecipe(cast<SelectInst>(Instr), R->operands(), *VPI,
*VPI, VPI->getDebugLoc());
if (Instruction::isCast(VPI->getOpcode())) {
auto *CastR = cast<VPInstructionWithType>(R);
auto *CI = cast<CastInst>(Instr);
auto *CastR = cast<VPInstructionWithType>(VPI);
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
CastR->getResultType(), *CI, *VPI);
CastR->getResultType(), CI, *VPI, *VPI,
VPI->getDebugLoc());
}
return tryToWiden(VPI);
@ -8269,8 +8273,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
SmallVector<VPValue *, 2> Ops;
Ops.push_back(Plan.getOrAddLiveIn(Zero));
Ops.push_back(BinOp);
BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(),
ReductionI->getDebugLoc());
BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRFlags(*ReductionI),
VPIRMetadata(), ReductionI->getDebugLoc());
Builder.insert(BinOp->getDefiningRecipe());
ReductionOpcode = Instruction::Add;
}
@ -8454,9 +8458,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
auto *VPI = cast<VPInstruction>(SingleDef);
auto *Recipe = new VPReplicateRecipe(
SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
*cast<VPInstruction>(SingleDef));
SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, *VPI,
*VPI, VPI->getDebugLoc());
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();

92
llvm/lib/Transforms/Vectorize/VPlan.h
View File

@ -882,14 +882,6 @@ public:
/// A pure-virtual common base class for recipes defining a single VPValue and
/// using IR flags.
struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
: VPSingleDefRecipe(SC, Operands, DL), VPIRFlags() {}
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
Instruction &I)
: VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()), VPIRFlags(I) {}
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags,
DebugLoc DL = DebugLoc::getUnknown())
@ -1474,9 +1466,12 @@ public:
VPIRMetadata(Metadata), Opcode(Opcode) {}
VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
const VPIRMetadata &Metadata, DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I),
VPIRMetadata(Metadata), Opcode(I.getOpcode()) {}
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
setUnderlyingValue(&I);
}
~VPWidenRecipe() override = default;
@ -1517,30 +1512,22 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
CastInst &UI, const VPIRMetadata &Metadata)
: VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(UI.getOpcode() == Opcode &&
"opcode of underlying cast doesn't match");
}
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
const VPIRFlags &Flags = {},
CastInst *CI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
setUnderlyingValue(CI);
}
~VPWidenCastRecipe() override = default;
VPWidenCastRecipe *clone() override {
auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this,
*this, getDebugLoc());
if (auto *UV = getUnderlyingValue())
New->setUnderlyingValue(UV);
return New;
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
cast_or_null<CastInst>(getUnderlyingValue()),
*this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
@ -1585,13 +1572,17 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, Flags,
DL),
VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
MayHaveSideEffects(CI.mayHaveSideEffects()) {
setUnderlyingValue(&CI);
}
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
@ -1617,7 +1608,7 @@ public:
VPWidenIntrinsicRecipe *clone() override {
if (Value *CI = getUnderlyingValue())
return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
operands(), ResultTy, *this,
operands(), ResultTy, *this, *this,
getDebugLoc());
return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
*this, *this, getDebugLoc());
@ -1671,10 +1662,11 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
public:
VPWidenCallRecipe(Value *UV, Function *Variant,
ArrayRef<VPValue *> CallArguments,
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
*cast<Instruction>(UV)),
VPIRMetadata(*cast<Instruction>(UV)), Variant(Variant) {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
assert(
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
@ -1684,7 +1676,7 @@ public:
VPWidenCallRecipe *clone() override {
return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(),
getDebugLoc());
*this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
@ -1761,16 +1753,19 @@ protected:
/// instruction.
struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags,
public VPIRMetadata {
VPWidenSelectRecipe(SelectInst &I, ArrayRef<VPValue *> Operands,
const VPIRMetadata &MD = {})
: VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I),
VPIRMetadata(MD) {}
VPWidenSelectRecipe(SelectInst *SI, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, Flags, DL),
VPIRMetadata(MD) {
setUnderlyingValue(SI);
}
~VPWidenSelectRecipe() override = default;
VPWidenSelectRecipe *clone() override {
return new VPWidenSelectRecipe(*cast<SelectInst>(getUnderlyingInstr()),
operands(), *this);
return new VPWidenSelectRecipe(cast<SelectInst>(getUnderlyingInstr()),
operands(), *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
@ -1822,9 +1817,12 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
}
public:
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands)
: VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, *GEP),
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, Flags, DL),
SourceElementTy(GEP->getSourceElementType()) {
setUnderlyingValue(GEP);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
(void)Metadata;
getMetadataToPropagate(GEP, Metadata);
@ -1835,7 +1833,7 @@ public:
VPWidenGEPRecipe *clone() override {
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(getUnderlyingInstr()),
operands());
operands(), *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC)
@ -2929,10 +2927,12 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
public:
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
bool IsSingleScalar, VPValue *Mask = nullptr,
VPIRMetadata Metadata = {})
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, Flags, DL),
VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
IsPredicated(Mask) {
setUnderlyingValue(I);
if (Mask)
addOperand(Mask);
}
@ -2940,9 +2940,9 @@ public:
~VPReplicateRecipe() override = default;
VPReplicateRecipe *clone() override {
auto *Copy =
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsSingleScalar,
isPredicated() ? getMask() : nullptr, *this);
auto *Copy = new VPReplicateRecipe(
getUnderlyingInstr(), operands(), IsSingleScalar,
isPredicated() ? getMask() : nullptr, *this, *this, getDebugLoc());
Copy->transferFlags(*this);
return Copy;
}

11
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
View File

@ -190,7 +190,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// recipes.
if (Br->isConditional()) {
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst,
VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, {},
VPIRMetadata(*Inst), Inst->getDebugLoc());
}
@ -205,7 +205,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())};
for (auto Case : SI->cases())
Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst,
VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, {},
VPIRMetadata(*Inst), Inst->getDebugLoc());
continue;
}
@ -255,13 +255,14 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
if (auto *CI = dyn_cast<CastInst>(Inst)) {
NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0],
CI->getType(), CI->getDebugLoc(),
{}, MD);
VPIRFlags(*CI), MD);
NewR->setUnderlyingValue(CI);
} else {
// Build VPInstruction for any arbitrary Instruction without specific
// representation in VPlan.
NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD,
Inst->getDebugLoc());
NewR =
VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst,
VPIRFlags(*Inst), MD, Inst->getDebugLoc());
}
}

16
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
View File

@ -2056,24 +2056,26 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
switch (OpType) {
case OperationType::OverflowingBinOp:
return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
Opcode == Instruction::Mul ||
Opcode == Instruction::Mul || Opcode == Instruction::Shl ||
Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;
case OperationType::Trunc:
return Opcode == Instruction::Trunc;
case OperationType::DisjointOp:
return Opcode == Instruction::Or;
case OperationType::PossiblyExactOp:
return Opcode == Instruction::AShr;
return Opcode == Instruction::AShr || Opcode == Instruction::LShr ||
Opcode == Instruction::UDiv || Opcode == Instruction::SDiv;
case OperationType::GEPOp:
return Opcode == Instruction::GetElementPtr ||
Opcode == VPInstruction::PtrAdd ||
Opcode == VPInstruction::WidePtrAdd;
case OperationType::FPMathOp:
return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
Opcode == Instruction::FSub || Opcode == Instruction::FNeg ||
Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc ||
Opcode == Instruction::FCmp || Opcode == Instruction::Select ||
return Opcode == Instruction::Call || Opcode == Instruction::FAdd ||
Opcode == Instruction::FMul || Opcode == Instruction::FSub ||
Opcode == Instruction::FNeg || Opcode == Instruction::FDiv ||
Opcode == Instruction::FRem || Opcode == Instruction::FPExt ||
Opcode == Instruction::FPTrunc || Opcode == Instruction::FCmp ||
Opcode == Instruction::Select ||
Opcode == VPInstruction::WideIVStep ||
Opcode == VPInstruction::ReductionStartVector ||
Opcode == VPInstruction::ComputeReductionResult;

50
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View File

@ -104,24 +104,26 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, *VPI,
Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands(), *VPI,
Ingredient.getDebugLoc());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI);
if (VectorID == Intrinsic::not_intrinsic)
return false;
NewRecipe = new VPWidenIntrinsicRecipe(
*CI, getVectorIntrinsicIDForCall(CI, &TLI),
drop_end(Ingredient.operands()), CI->getType(), *VPI,
CI->getDebugLoc());
drop_end(Ingredient.operands()), CI->getType(), VPIRFlags(*CI),
*VPI, CI->getDebugLoc());
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands(), *VPI);
NewRecipe = new VPWidenSelectRecipe(SI, Ingredient.operands(), *VPI,
*VPI, Ingredient.getDebugLoc());
} else if (auto *CI = dyn_cast<CastInst>(Inst)) {
NewRecipe =
new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0),
CI->getType(), *CI, *VPI);
NewRecipe = new VPWidenCastRecipe(
CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), CI,
VPIRFlags(*CI), VPIRMetadata(*CI));
} else {
NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
Ingredient.getDebugLoc());
*VPI, Ingredient.getDebugLoc());
}
}
@ -226,7 +228,8 @@ static bool sinkScalarOperands(VPlan &Plan) {
// then cloning should be sufficient here.
Instruction *I = SinkCandidate->getUnderlyingInstr();
Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true,
nullptr /*Mask*/, *SinkCandidateRepR);
nullptr /*Mask*/, *SinkCandidateRepR,
*SinkCandidateRepR);
// TODO: add ".cloned" suffix to name of Clone's VPValue.
} else {
Clone = SinkCandidate->clone();
@ -385,7 +388,8 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
// mask but in the replicate region.
auto *RecipeWithoutMask = new VPReplicateRecipe(
PredRecipe->getUnderlyingInstr(), drop_end(PredRecipe->operands()),
PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe);
PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
PredRecipe->getDebugLoc());
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
@ -691,7 +695,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
// analysis.
auto Users = collectUsersRecursively(PhiR);
for (VPUser *U : reverse(Users)) {
auto *Def = dyn_cast<VPSingleDefRecipe>(U);
auto *Def = dyn_cast<VPRecipeWithIRFlags>(U);
auto *RepR = dyn_cast<VPReplicateRecipe>(U);
// Skip recipes that shouldn't be narrowed.
if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
@ -704,7 +708,8 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
continue;
auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(),
Def->operands(), /*IsUniform*/ true);
Def->operands(), /*IsUniform*/ true,
/*Mask*/ nullptr, /*Flags*/ *Def);
Clone->insertAfter(Def);
Def->replaceAllUsesWith(Clone);
}
@ -1423,12 +1428,13 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
continue;
auto *RepOrWidenR = cast<VPSingleDefRecipe>(&R);
auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(&R);
if (RepR && isa<StoreInst>(RepR->getUnderlyingInstr()) &&
vputils::isSingleScalar(RepR->getOperand(1))) {
auto *Clone = new VPReplicateRecipe(
RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Metadata*/);
true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
*RepR /*Metadata*/, RepR->getDebugLoc());
Clone->insertBefore(RepOrWidenR);
unsigned ExtractOpc =
vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1))
@ -1469,9 +1475,9 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
}))
continue;
auto *Clone = new VPReplicateRecipe(RepOrWidenR->getUnderlyingInstr(),
RepOrWidenR->operands(),
true /*IsSingleScalar*/);
auto *Clone = new VPReplicateRecipe(
RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
true /*IsSingleScalar*/, nullptr, *RepOrWidenR);
Clone->insertBefore(RepOrWidenR);
RepOrWidenR->replaceAllUsesWith(Clone);
if (isDeadRecipe(*RepOrWidenR))
@ -3824,15 +3830,15 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
Ext0->getOpcode() == Ext1->getOpcode() &&
IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) {
auto *NewExt0 = new VPWidenCastRecipe(
Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0,
*Ext0, Ext0->getDebugLoc());
Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), nullptr,
*Ext0, *Ext0, Ext0->getDebugLoc());
NewExt0->insertBefore(Ext0);
VPWidenCastRecipe *NewExt1 = NewExt0;
if (Ext0 != Ext1) {
NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0),
Ext->getResultType(), *Ext1, *Ext1,
Ext1->getDebugLoc());
Ext->getResultType(), nullptr, *Ext1,
*Ext1, Ext1->getDebugLoc());
NewExt1->insertBefore(Ext1);
}
Mul->setOperand(0, NewExt0);
@ -4353,7 +4359,7 @@ narrowInterleaveGroupOp(VPValue *V, SmallPtrSetImpl<VPValue *> &NarrowedOps) {
// process one original iteration.
auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
/*IsUniform*/ true,
/*Mask*/ nullptr, *WideLoad);
/*Mask*/ nullptr, {}, *WideLoad);
N->insertBefore(WideLoad);
NarrowedOps.insert(N);
return N;

6
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
View File

@ -518,9 +518,9 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
// TODO: have cloning of replicate recipes also provide the desired result
// coupled with setting its operands to NewOps (deriving IsSingleScalar and
// Mask from the operands?)
New =
new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps,
/*IsSingleScalar=*/true, /*Mask=*/nullptr, *RepR);
New = new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps,
/*IsSingleScalar=*/true, /*Mask=*/nullptr,
*RepR, *RepR, RepR->getDebugLoc());
} else {
assert(isa<VPInstruction>(DefR) &&
"DefR must be a VPReplicateRecipe or VPInstruction");

14
llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
View File

@ -14,23 +14,23 @@ define void @foo(i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: outer.header:
; CHECK-NEXT: EMIT-SCALAR ir<%outer.iv> = phi [ ir<%outer.iv.next>, outer.latch ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr inbounds ir<@arr2>, ir<0>, ir<%outer.iv>
; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1>
; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
; CHECK-NEXT: EMIT ir<%add> = add nsw ir<%outer.iv>, ir<%n>
; CHECK-NEXT: Successor(s): inner
; CHECK-EMPTY:
; CHECK-NEXT: inner:
; CHECK-NEXT: EMIT-SCALAR ir<%inner.iv> = phi [ ir<%inner.iv.next>, inner ], [ ir<0>, outer.header ]
; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr inbounds ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2>
; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
; CHECK-NEXT: EMIT ir<%inner.iv.next> = add nuw nsw ir<%inner.iv>, ir<1>
; CHECK-NEXT: EMIT ir<%inner.ec> = icmp eq ir<%inner.iv.next>, ir<8>
; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec>
; CHECK-NEXT: Successor(s): outer.latch, inner
; CHECK-EMPTY:
; CHECK-NEXT: outer.latch:
; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
; CHECK-NEXT: EMIT ir<%outer.iv.next> = add nuw nsw ir<%outer.iv>, ir<1>
; CHECK-NEXT: EMIT ir<%outer.ec> = icmp eq ir<%outer.iv.next>, ir<8>
; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
; CHECK-NEXT: Successor(s): ir-bb<exit>, outer.header
; CHECK-EMPTY:

10
llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
View File

@ -139,12 +139,12 @@ compound=true
"vector.body:\l" +
" EMIT vp\<%2\> = CANONICAL-INDUCTION ir\<0\>, vp\<%index.next\>\l" +
" EMIT-SCALAR ir\<%indvars.iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%indvars.iv.next\>, vector.body ]\l" +
" EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%indvars.iv\>\l" +
" EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%indvars.iv\>\l" +
" EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" +
" EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" +
" EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
" EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\>, ir\<1\>\l" +
" EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\>, ir\<%N\>\l" +
" EMIT ir\<%exitcond\> = icmp ne ir\<%indvars.iv.next\>, ir\<%N\>\l" +
" EMIT vp\<%3\> = not ir\<%exitcond\>\l" +
" EMIT vp\<%index.next\> = add nuw vp\<%2\>, vp\<%0\>\l" +
" EMIT branch-on-count vp\<%index.next\>, vp\<%1\>\l" +
@ -305,9 +305,9 @@ compound=true
"vector.body:\l" +
" EMIT vp\<%2\> = CANONICAL-INDUCTION ir\<0\>, vp\<%index.next\>\l" +
" EMIT-SCALAR ir\<%iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%iv.next\>, loop.latch ]\l" +
" EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%iv\>\l" +
" EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%iv\>\l" +
" EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" +
" EMIT ir\<%c\> = icmp ir\<%l1\>, ir\<0\>\l" +
" EMIT ir\<%c\> = icmp eq ir\<%l1\>, ir\<0\>\l" +
"Successor(s): loop.latch\l"
]
N4 -> N6 [ label=""]
@ -316,7 +316,7 @@ compound=true
" EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" +
" EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
" EMIT ir\<%iv.next\> = add ir\<%iv\>, ir\<1\>\l" +
" EMIT ir\<%exitcond\> = icmp ir\<%iv.next\>, ir\<%N\>\l" +
" EMIT ir\<%exitcond\> = icmp ne ir\<%iv.next\>, ir\<%N\>\l" +
" EMIT vp\<%3\> = not ir\<%exitcond\>\l" +
" EMIT vp\<%index.next\> = add nuw vp\<%2\>, vp\<%0\>\l" +
" EMIT branch-on-count vp\<%index.next\>, vp\<%1\>\l" +

15
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
View File

@ -1009,7 +1009,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
VPWidenRecipe WidenR(*AI, Args, VPIRMetadata(), DebugLoc());
VPWidenRecipe WidenR(*AI, Args);
checkVPRecipeCastImpl<VPWidenRecipe, VPUser, VPIRMetadata>(&WidenR);
delete AI;
@ -1053,7 +1053,7 @@ TEST_F(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(Op3);
VPWidenSelectRecipe WidenSelectR(*SelectI,
VPWidenSelectRecipe WidenSelectR(SelectI,
make_range(Args.begin(), Args.end()));
checkVPRecipeCastImpl<VPWidenSelectRecipe, VPUser, VPIRMetadata>(
@ -1093,7 +1093,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) {
IntegerType *Int64 = IntegerType::get(C, 64);
auto *Cast = CastInst::CreateZExtOrBitCast(PoisonValue::get(Int32), Int64);
VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast, {});
VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, Cast);
checkVPRecipeCastImpl<VPWidenCastRecipe, VPUser, VPIRMetadata>(&Recipe);
delete Cast;
@ -1264,7 +1264,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
VPWidenRecipe Recipe(*AI, Args, VPIRMetadata(), DebugLoc());
VPWidenRecipe Recipe(*AI, Args);
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@ -1283,7 +1283,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(Op3);
VPWidenSelectRecipe Recipe(*SelectI, make_range(Args.begin(), Args.end()));
VPWidenSelectRecipe Recipe(SelectI, make_range(Args.begin(), Args.end()));
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@ -1412,7 +1412,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(CalledFn);
VPWidenCallRecipe Recipe(Call, TheFn, Args);
VPWidenCallRecipe Recipe(Call, TheFn, Args, VPIRFlags(), VPIRMetadata());
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@ -1468,8 +1468,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) {
VPValue *ExtVPV2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
Args.push_back(ExtVPV1);
Args.push_back(ExtVPV2);
VPWidenRecipe *WidenR =
new VPWidenRecipe(*AI, Args, VPIRMetadata(), DebugLoc());
VPWidenRecipe *WidenR = new VPWidenRecipe(*AI, Args);
VPBB1->appendRecipe(WidenR);
{