diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 6e540dc7a5f3..082489f70f1c 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8234,11 +8234,11 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R, if ((Recipe = tryToOptimizeInductionPHI(PhiR))) return Recipe; - VPHeaderPHIRecipe *PhiRecipe = nullptr; assert((Legal->isReductionVariable(Phi) || Legal->isFixedOrderRecurrence(Phi)) && "can only widen reductions and fixed-order recurrences here"); VPValue *StartV = R->getOperand(0); + VPValue *BackedgeValue = R->getOperand(1); if (Legal->isReductionVariable(Phi)) { const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi); assert(RdxDesc.getRecurrenceStartValue() == @@ -8250,22 +8250,20 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R, unsigned ScaleFactor = getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1); - PhiRecipe = new VPReductionPHIRecipe( - Phi, RdxDesc.getRecurrenceKind(), *StartV, + return new VPReductionPHIRecipe( + Phi, RdxDesc.getRecurrenceKind(), *StartV, *BackedgeValue, getReductionStyle(UseInLoopReduction, UseOrderedReductions, ScaleFactor), RdxDesc.hasUsesOutsideReductionChain()); - } else { - // TODO: Currently fixed-order recurrences are modeled as chains of - // first-order recurrences. If there are no users of the intermediate - // recurrences in the chain, the fixed order recurrence should be modeled - // directly, enabling more efficient codegen. - PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV); } - // Add backedge value. - PhiRecipe->addOperand(R->getOperand(1)); - return PhiRecipe; + + // TODO: Currently fixed-order recurrences are modeled as chains of + // first-order recurrences. If there are no users of the intermediate + // recurrences in the chain, the fixed order recurrence should be modeled + // directly, enabling more efficient codegen. + return new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV, *BackedgeValue); } + assert(!R->isPhi() && "only VPPhi nodes expected at this point"); auto *VPI = cast(R); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 0f8f6abab7b0..60318960353d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2366,14 +2366,17 @@ protected: /// first operand of the recipe and the incoming value from the backedge is the /// second operand. struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe { - VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start) - : VPHeaderPHIRecipe(VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) {} + VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start, + VPValue &BackedgeValue) + : VPHeaderPHIRecipe(VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) { + addOperand(&BackedgeValue); + } VP_CLASSOF_IMPL(VPDef::VPFirstOrderRecurrencePHISC) VPFirstOrderRecurrencePHIRecipe *clone() override { return new VPFirstOrderRecurrencePHIRecipe( - cast(getUnderlyingInstr()), *getOperand(0)); + cast(getUnderlyingInstr()), *getOperand(0), *getOperand(1)); } void execute(VPTransformState &State) override; @@ -2439,20 +2442,21 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public: /// Create a new VPReductionPHIRecipe for the reduction \p Phi. VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start, - ReductionStyle Style, + VPValue &BackedgeValue, ReductionStyle Style, bool HasUsesOutsideReductionChain = false) : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind), Style(Style), - HasUsesOutsideReductionChain(HasUsesOutsideReductionChain) {} + HasUsesOutsideReductionChain(HasUsesOutsideReductionChain) { + addOperand(&BackedgeValue); + } ~VPReductionPHIRecipe() override = default; VPReductionPHIRecipe *clone() override { - auto *R = new VPReductionPHIRecipe( + return new VPReductionPHIRecipe( dyn_cast_or_null(getUnderlyingValue()), getRecurrenceKind(), - *getOperand(0), Style, HasUsesOutsideReductionChain); - R->addOperand(getBackedgeValue()); - return R; + *getOperand(0), *getBackedgeValue(), Style, + HasUsesOutsideReductionChain); } VP_CLASSOF_IMPL(VPDef::VPReductionPHISC)