llvm-project/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
Jonas Paulsson d3081aafc4
[SystemZ, LoopVectorizer] Enable vectorization of epilogue loops. (#172925)
This enables vectorization of epilogue loops produced by LoopVectorizer on
SystemZ.

LoopVectorizationCostModel::isEpilogueVectorizationProfitable() and
TTI.preferEpilogueVectorization() have been refactored slightly so that
targets can override preferEpilogueVectorization(ElementCount Iters) and
directly control this, whereas before this depended on
TTI.getMaxInterleaveFactor() as well.

The Iters passed to preferEpilogueVectorization() reflects the total number
of scalar iterations performed in the vectorized loop (including interleaving).

The default implementation of preferEpilogueVectorization() now subsumes
the old check against getMaxInterleaveFactor(). This patch should be NFC for
other targets.
2026-02-22 10:59:09 -06:00

165 lines
7.1 KiB
C++

//===-- SystemZTargetTransformInfo.h - SystemZ-specific TTI ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZTARGETTRANSFORMINFO_H
#define LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZTARGETTRANSFORMINFO_H
#include "SystemZTargetMachine.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/BasicTTIImpl.h"
namespace llvm {
class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
typedef BasicTTIImplBase<SystemZTTIImpl> BaseT;
typedef TargetTransformInfo TTI;
friend BaseT;
const SystemZSubtarget *ST;
const SystemZTargetLowering *TLI;
const SystemZSubtarget *getST() const { return ST; }
const SystemZTargetLowering *getTLI() const { return TLI; }
unsigned const LIBCALL_COST = 30;
bool isInt128InVR(Type *Ty) const {
return Ty->isIntegerTy(128) && ST->hasVector();
}
public:
explicit SystemZTTIImpl(const SystemZTargetMachine *TM, const Function &F)
: BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
TLI(ST->getTargetLowering()) {}
/// \name Scalar TTI Implementations
/// @{
unsigned adjustInliningThreshold(const CallBase *CB) const override;
InstructionCost getIntImmCost(const APInt &Imm, Type *Ty,
TTI::TargetCostKind CostKind) const override;
InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx,
const APInt &Imm, Type *Ty,
TTI::TargetCostKind CostKind,
Instruction *Inst = nullptr) const override;
InstructionCost
getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm,
Type *Ty, TTI::TargetCostKind CostKind) const override;
TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override;
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE) const override;
void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
TTI::PeelingPreferences &PP) const override;
bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
const TargetTransformInfo::LSRCost &C2) const override;
/// @}
/// \name Vector TTI Implementations
/// @{
unsigned getNumberOfRegisters(unsigned ClassID) const override;
TypeSize
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;
unsigned getCacheLineSize() const override { return 256; }
unsigned getPrefetchDistance() const override { return 4500; }
unsigned getMinPrefetchStride(unsigned NumMemAccesses,
unsigned NumStridedMemAccesses,
unsigned NumPrefetches,
bool HasCall) const override;
bool enableWritePrefetching() const override { return true; }
bool hasDivRemOp(Type *DataType, bool IsSigned) const override;
bool prefersVectorizedAddressing() const override { return false; }
bool LSRWithInstrQueries() const override { return true; }
InstructionCost
getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
bool Insert, bool Extract,
TTI::TargetCostKind CostKind,
bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
TTI::VectorInstrContext VIC =
TTI::VectorInstrContext::None) const override;
bool supportsEfficientVectorElementLoadStore() const override { return true; }
bool enableInterleavedAccessVectorization() const override { return true; }
InstructionCost getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
InstructionCost
getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args = {},
const Instruction *CxtI = nullptr) const override;
unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy) const;
unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) const;
unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
const Instruction *I) const;
InstructionCost
getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
InstructionCost getCmpSelInstrCost(
unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
using BaseT::getVectorInstrCost;
InstructionCost
getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
unsigned Index, const Value *Op0, const Value *Op1,
TTI::VectorInstrContext VIC =
TTI::VectorInstrContext::None) const override;
bool isFoldableLoad(const LoadInst *Ld,
const Instruction *&FoldedValue) const;
InstructionCost getMemoryOpCost(
unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
TTI::TargetCostKind CostKind,
TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
const Instruction *I = nullptr) const override;
InstructionCost getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond = false, bool UseMaskForGaps = false) const override;
InstructionCost
getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
std::optional<FastMathFlags> FMF,
TTI::TargetCostKind CostKind) const override;
InstructionCost
getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
TTI::TargetCostKind CostKind) const override;
InstructionCost
getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
TTI::TargetCostKind CostKind) const override;
bool preferEpilogueVectorization(ElementCount Iters) const override {
return true;
}
bool shouldExpandReduction(const IntrinsicInst *II) const override;
/// @}
};
} // end namespace llvm
#endif