8ec406757c
This patch implements explicit unrolling by UF as VPlan transform. In follow up patches this will allow simplifying VPTransform state (no need to store unrolled parts) as well as recipe execution (no need to generate code for multiple parts in an each recipe). It also allows for more general optimziations (e.g. avoid generating code for recipes that are uniform-across parts). It also unifies the logic dealing with unrolled parts in a single place, rather than spreading it out across multiple places (e.g. VPlan post processing for header-phi recipes previously.) In the initial implementation, a number of recipes still take the unrolled part as additional, optional argument, if their execution depends on the unrolled part. The computation for start/step values for scalable inductions changed slightly. Previously the step would be computed as scalar and then splatted, now vscale gets splatted and multiplied by the step in a vector mul. This has been split off https://github.com/llvm/llvm-project/pull/94339 which also includes changes to simplify VPTransfomState and recipes' ::execute. The current version mostly leaves existing ::execute untouched and instead sets VPTransfomState::UF to 1. A follow-up patch will clean up all references to VPTransformState::UF. Another follow-up patch will simplify VPTransformState to only store a single vector value per VPValue. PR: https://github.com/llvm/llvm-project/pull/95842
118 lines
4.4 KiB
C++
118 lines
4.4 KiB
C++
//===- VPlanUtils.cpp - VPlan-related utilities ---------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "VPlanUtils.h"
|
|
#include "VPlanPatternMatch.h"
|
|
#include "llvm/ADT/TypeSwitch.h"
|
|
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
|
|
|
|
using namespace llvm;
|
|
|
|
bool vputils::onlyFirstLaneUsed(const VPValue *Def) {
|
|
return all_of(Def->users(),
|
|
[Def](const VPUser *U) { return U->onlyFirstLaneUsed(Def); });
|
|
}
|
|
|
|
bool vputils::onlyFirstPartUsed(const VPValue *Def) {
|
|
return all_of(Def->users(),
|
|
[Def](const VPUser *U) { return U->onlyFirstPartUsed(Def); });
|
|
}
|
|
|
|
VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
|
|
ScalarEvolution &SE) {
|
|
if (auto *Expanded = Plan.getSCEVExpansion(Expr))
|
|
return Expanded;
|
|
VPValue *Expanded = nullptr;
|
|
if (auto *E = dyn_cast<SCEVConstant>(Expr))
|
|
Expanded = Plan.getOrAddLiveIn(E->getValue());
|
|
else if (auto *E = dyn_cast<SCEVUnknown>(Expr))
|
|
Expanded = Plan.getOrAddLiveIn(E->getValue());
|
|
else {
|
|
Expanded = new VPExpandSCEVRecipe(Expr, SE);
|
|
Plan.getPreheader()->appendRecipe(Expanded->getDefiningRecipe());
|
|
}
|
|
Plan.addSCEVExpansion(Expr, Expanded);
|
|
return Expanded;
|
|
}
|
|
|
|
bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) {
|
|
if (isa<VPActiveLaneMaskPHIRecipe>(V))
|
|
return true;
|
|
|
|
auto IsWideCanonicalIV = [](VPValue *A) {
|
|
return isa<VPWidenCanonicalIVRecipe>(A) ||
|
|
(isa<VPWidenIntOrFpInductionRecipe>(A) &&
|
|
cast<VPWidenIntOrFpInductionRecipe>(A)->isCanonical());
|
|
};
|
|
|
|
VPValue *A, *B;
|
|
using namespace VPlanPatternMatch;
|
|
|
|
if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B))))
|
|
return B == Plan.getTripCount() &&
|
|
(match(A, m_ScalarIVSteps(m_CanonicalIV(), m_SpecificInt(1))) ||
|
|
IsWideCanonicalIV(A));
|
|
|
|
return match(V, m_Binary<Instruction::ICmp>(m_VPValue(A), m_VPValue(B))) &&
|
|
IsWideCanonicalIV(A) && B == Plan.getOrCreateBackedgeTakenCount();
|
|
}
|
|
|
|
const SCEV *vputils::getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE) {
|
|
if (V->isLiveIn())
|
|
return SE.getSCEV(V->getLiveInIRValue());
|
|
|
|
// TODO: Support constructing SCEVs for more recipes as needed.
|
|
return TypeSwitch<const VPRecipeBase *, const SCEV *>(V->getDefiningRecipe())
|
|
.Case<VPExpandSCEVRecipe>(
|
|
[](const VPExpandSCEVRecipe *R) { return R->getSCEV(); })
|
|
.Default([&SE](const VPRecipeBase *) { return SE.getCouldNotCompute(); });
|
|
}
|
|
|
|
bool vputils::isUniformAcrossVFsAndUFs(VPValue *V) {
|
|
using namespace VPlanPatternMatch;
|
|
// Live-ins are uniform.
|
|
if (V->isLiveIn())
|
|
return true;
|
|
|
|
VPRecipeBase *R = V->getDefiningRecipe();
|
|
if (R && V->isDefinedOutsideLoopRegions()) {
|
|
if (match(V->getDefiningRecipe(),
|
|
m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>(
|
|
m_VPValue())))
|
|
return false;
|
|
return all_of(R->operands(),
|
|
[](VPValue *Op) { return isUniformAcrossVFsAndUFs(Op); });
|
|
}
|
|
|
|
auto *CanonicalIV = R->getParent()->getPlan()->getCanonicalIV();
|
|
// Canonical IV chain is uniform.
|
|
if (V == CanonicalIV || V == CanonicalIV->getBackedgeValue())
|
|
return true;
|
|
|
|
return TypeSwitch<const VPRecipeBase *, bool>(R)
|
|
.Case<VPDerivedIVRecipe>([](const auto *R) { return true; })
|
|
.Case<VPReplicateRecipe>([](const auto *R) {
|
|
// Loads and stores that are uniform across VF lanes are handled by
|
|
// VPReplicateRecipe.IsUniform. They are also uniform across UF parts if
|
|
// all their operands are invariant.
|
|
// TODO: Further relax the restrictions.
|
|
return R->isUniform() &&
|
|
(isa<LoadInst, StoreInst>(R->getUnderlyingValue())) &&
|
|
all_of(R->operands(),
|
|
[](VPValue *Op) { return isUniformAcrossVFsAndUFs(Op); });
|
|
})
|
|
.Case<VPScalarCastRecipe, VPWidenCastRecipe>([](const auto *R) {
|
|
// A cast is uniform according to its operand.
|
|
return isUniformAcrossVFsAndUFs(R->getOperand(0));
|
|
})
|
|
.Default([](const VPRecipeBase *) { // A value is considered non-uniform
|
|
// unless proven otherwise.
|
|
return false;
|
|
});
|
|
}
|