f5889e59e6
Regbanklegalize rules for INTRIN_IMAGE loads and stores. Because of very large number of different type signatures, rule specifies only function for lowering (waterfall lowering of RsrcIdx operand if needed) and this function also applies register banks.
1497 lines
66 KiB
C++
1497 lines
66 KiB
C++
//===-- AMDGPURegBankLegalizeRules.cpp ------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
/// Definitions of RegBankLegalize Rules for all opcodes.
|
|
/// Implementation of container for all the Rules and search.
|
|
/// Fast search for most common case when Rule.Predicate checks LLT and
|
|
/// uniformity of register in operand 0.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "AMDGPURegBankLegalizeRules.h"
|
|
#include "AMDGPUInstrInfo.h"
|
|
#include "GCNSubtarget.h"
|
|
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
|
|
#include "llvm/CodeGen/MachineUniformityAnalysis.h"
|
|
#include "llvm/IR/IntrinsicsAMDGPU.h"
|
|
#include "llvm/Support/AMDGPUAddrSpace.h"
|
|
|
|
#define DEBUG_TYPE "amdgpu-regbanklegalize"
|
|
|
|
using namespace llvm;
|
|
using namespace AMDGPU;
|
|
|
|
bool AMDGPU::isAnyPtr(LLT Ty, unsigned Width) {
|
|
return Ty.isPointer() && Ty.getSizeInBits() == Width;
|
|
}
|
|
|
|
RegBankLLTMapping::RegBankLLTMapping(
|
|
std::initializer_list<RegBankLLTMappingApplyID> DstOpMappingList,
|
|
std::initializer_list<RegBankLLTMappingApplyID> SrcOpMappingList,
|
|
LoweringMethodID LoweringMethod)
|
|
: DstOpMapping(DstOpMappingList), SrcOpMapping(SrcOpMappingList),
|
|
LoweringMethod(LoweringMethod) {}
|
|
|
|
PredicateMapping::PredicateMapping(
|
|
std::initializer_list<UniformityLLTOpPredicateID> OpList,
|
|
std::function<bool(const MachineInstr &)> TestFunc)
|
|
: OpUniformityAndTypes(OpList), TestFunc(TestFunc) {}
|
|
|
|
bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
|
|
const MachineUniformityInfo &MUI,
|
|
const MachineRegisterInfo &MRI) {
|
|
switch (UniID) {
|
|
case S1:
|
|
return MRI.getType(Reg) == LLT::scalar(1);
|
|
case S16:
|
|
return MRI.getType(Reg) == LLT::scalar(16);
|
|
case S32:
|
|
return MRI.getType(Reg) == LLT::scalar(32);
|
|
case S64:
|
|
return MRI.getType(Reg) == LLT::scalar(64);
|
|
case S128:
|
|
return MRI.getType(Reg) == LLT::scalar(128);
|
|
case P0:
|
|
return MRI.getType(Reg) == LLT::pointer(0, 64);
|
|
case P1:
|
|
return MRI.getType(Reg) == LLT::pointer(1, 64);
|
|
case P2:
|
|
return MRI.getType(Reg) == LLT::pointer(2, 32);
|
|
case P3:
|
|
return MRI.getType(Reg) == LLT::pointer(3, 32);
|
|
case P4:
|
|
return MRI.getType(Reg) == LLT::pointer(4, 64);
|
|
case P5:
|
|
return MRI.getType(Reg) == LLT::pointer(5, 32);
|
|
case P8:
|
|
return MRI.getType(Reg) == LLT::pointer(8, 128);
|
|
case Ptr32:
|
|
return isAnyPtr(MRI.getType(Reg), 32);
|
|
case Ptr64:
|
|
return isAnyPtr(MRI.getType(Reg), 64);
|
|
case Ptr128:
|
|
return isAnyPtr(MRI.getType(Reg), 128);
|
|
case V2S16:
|
|
return MRI.getType(Reg) == LLT::fixed_vector(2, 16);
|
|
case V2S32:
|
|
return MRI.getType(Reg) == LLT::fixed_vector(2, 32);
|
|
case V3S32:
|
|
return MRI.getType(Reg) == LLT::fixed_vector(3, 32);
|
|
case V4S32:
|
|
return MRI.getType(Reg) == LLT::fixed_vector(4, 32);
|
|
case B32:
|
|
return MRI.getType(Reg).getSizeInBits() == 32;
|
|
case B64:
|
|
return MRI.getType(Reg).getSizeInBits() == 64;
|
|
case B96:
|
|
return MRI.getType(Reg).getSizeInBits() == 96;
|
|
case B128:
|
|
return MRI.getType(Reg).getSizeInBits() == 128;
|
|
case B160:
|
|
return MRI.getType(Reg).getSizeInBits() == 160;
|
|
case B256:
|
|
return MRI.getType(Reg).getSizeInBits() == 256;
|
|
case B512:
|
|
return MRI.getType(Reg).getSizeInBits() == 512;
|
|
case UniS1:
|
|
return MRI.getType(Reg) == LLT::scalar(1) && MUI.isUniform(Reg);
|
|
case UniS16:
|
|
return MRI.getType(Reg) == LLT::scalar(16) && MUI.isUniform(Reg);
|
|
case UniS32:
|
|
return MRI.getType(Reg) == LLT::scalar(32) && MUI.isUniform(Reg);
|
|
case UniS64:
|
|
return MRI.getType(Reg) == LLT::scalar(64) && MUI.isUniform(Reg);
|
|
case UniS128:
|
|
return MRI.getType(Reg) == LLT::scalar(128) && MUI.isUniform(Reg);
|
|
case UniP0:
|
|
return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isUniform(Reg);
|
|
case UniP1:
|
|
return MRI.getType(Reg) == LLT::pointer(1, 64) && MUI.isUniform(Reg);
|
|
case UniP2:
|
|
return MRI.getType(Reg) == LLT::pointer(2, 32) && MUI.isUniform(Reg);
|
|
case UniP3:
|
|
return MRI.getType(Reg) == LLT::pointer(3, 32) && MUI.isUniform(Reg);
|
|
case UniP4:
|
|
return MRI.getType(Reg) == LLT::pointer(4, 64) && MUI.isUniform(Reg);
|
|
case UniP5:
|
|
return MRI.getType(Reg) == LLT::pointer(5, 32) && MUI.isUniform(Reg);
|
|
case UniP8:
|
|
return MRI.getType(Reg) == LLT::pointer(8, 128) && MUI.isUniform(Reg);
|
|
case UniPtr32:
|
|
return isAnyPtr(MRI.getType(Reg), 32) && MUI.isUniform(Reg);
|
|
case UniPtr64:
|
|
return isAnyPtr(MRI.getType(Reg), 64) && MUI.isUniform(Reg);
|
|
case UniPtr128:
|
|
return isAnyPtr(MRI.getType(Reg), 128) && MUI.isUniform(Reg);
|
|
case UniV2S16:
|
|
return MRI.getType(Reg) == LLT::fixed_vector(2, 16) && MUI.isUniform(Reg);
|
|
case UniV2S32:
|
|
return MRI.getType(Reg) == LLT::fixed_vector(2, 32) && MUI.isUniform(Reg);
|
|
case UniB32:
|
|
return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isUniform(Reg);
|
|
case UniB64:
|
|
return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isUniform(Reg);
|
|
case UniB96:
|
|
return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isUniform(Reg);
|
|
case UniB128:
|
|
return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isUniform(Reg);
|
|
case UniB160:
|
|
return MRI.getType(Reg).getSizeInBits() == 160 && MUI.isUniform(Reg);
|
|
case UniB256:
|
|
return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isUniform(Reg);
|
|
case UniB512:
|
|
return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isUniform(Reg);
|
|
case UniBRC: {
|
|
if (!MUI.isUniform(Reg))
|
|
return false;
|
|
// Check if there is SGPR register class of same size as the LLT.
|
|
const SIRegisterInfo *TRI =
|
|
static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
|
|
// There is no 16 bit SGPR register class. Extra size check is required
|
|
// since getSGPRClassForBitWidth returns SReg_32RegClass for Size 16.
|
|
unsigned LLTSize = MRI.getType(Reg).getSizeInBits();
|
|
return LLTSize >= 32 && TRI->getSGPRClassForBitWidth(LLTSize);
|
|
}
|
|
case DivS1:
|
|
return MRI.getType(Reg) == LLT::scalar(1) && MUI.isDivergent(Reg);
|
|
case DivS16:
|
|
return MRI.getType(Reg) == LLT::scalar(16) && MUI.isDivergent(Reg);
|
|
case DivS32:
|
|
return MRI.getType(Reg) == LLT::scalar(32) && MUI.isDivergent(Reg);
|
|
case DivS64:
|
|
return MRI.getType(Reg) == LLT::scalar(64) && MUI.isDivergent(Reg);
|
|
case DivS128:
|
|
return MRI.getType(Reg) == LLT::scalar(128) && MUI.isDivergent(Reg);
|
|
case DivP0:
|
|
return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isDivergent(Reg);
|
|
case DivP1:
|
|
return MRI.getType(Reg) == LLT::pointer(1, 64) && MUI.isDivergent(Reg);
|
|
case DivP2:
|
|
return MRI.getType(Reg) == LLT::pointer(2, 32) && MUI.isDivergent(Reg);
|
|
case DivP3:
|
|
return MRI.getType(Reg) == LLT::pointer(3, 32) && MUI.isDivergent(Reg);
|
|
case DivP4:
|
|
return MRI.getType(Reg) == LLT::pointer(4, 64) && MUI.isDivergent(Reg);
|
|
case DivP5:
|
|
return MRI.getType(Reg) == LLT::pointer(5, 32) && MUI.isDivergent(Reg);
|
|
case DivPtr32:
|
|
return isAnyPtr(MRI.getType(Reg), 32) && MUI.isDivergent(Reg);
|
|
case DivPtr64:
|
|
return isAnyPtr(MRI.getType(Reg), 64) && MUI.isDivergent(Reg);
|
|
case DivPtr128:
|
|
return isAnyPtr(MRI.getType(Reg), 128) && MUI.isDivergent(Reg);
|
|
case DivV2S16:
|
|
return MRI.getType(Reg) == LLT::fixed_vector(2, 16) && MUI.isDivergent(Reg);
|
|
case DivV2S32:
|
|
return MRI.getType(Reg) == LLT::fixed_vector(2, 32) && MUI.isDivergent(Reg);
|
|
case DivB32:
|
|
return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isDivergent(Reg);
|
|
case DivB64:
|
|
return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isDivergent(Reg);
|
|
case DivB96:
|
|
return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isDivergent(Reg);
|
|
case DivB128:
|
|
return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isDivergent(Reg);
|
|
case DivB160:
|
|
return MRI.getType(Reg).getSizeInBits() == 160 && MUI.isDivergent(Reg);
|
|
case DivB256:
|
|
return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isDivergent(Reg);
|
|
case DivB512:
|
|
return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isDivergent(Reg);
|
|
case DivBRC: {
|
|
if (!MUI.isDivergent(Reg))
|
|
return false;
|
|
// Check if there is VGPR register class of same size as the LLT.
|
|
const SIRegisterInfo *TRI =
|
|
static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
|
|
return TRI->getSGPRClassForBitWidth(MRI.getType(Reg).getSizeInBits());
|
|
}
|
|
case _:
|
|
return true;
|
|
default:
|
|
llvm_unreachable("missing matchUniformityAndLLT");
|
|
}
|
|
}
|
|
|
|
bool PredicateMapping::match(const MachineInstr &MI,
|
|
const MachineUniformityInfo &MUI,
|
|
const MachineRegisterInfo &MRI) const {
|
|
// Check LLT signature.
|
|
for (unsigned i = 0; i < OpUniformityAndTypes.size(); ++i) {
|
|
const MachineOperand &MO = MI.getOperand(i);
|
|
if (OpUniformityAndTypes[i] == _) {
|
|
assert((!MI.getOperand(i).isReg() ||
|
|
!MI.getOperand(i).getReg().isVirtual()) &&
|
|
"_ is for non-register and physical register operands only");
|
|
continue;
|
|
}
|
|
|
|
// Remaining IDs check registers.
|
|
if (!MO.isReg())
|
|
return false;
|
|
|
|
if (!matchUniformityAndLLT(MO.getReg(), OpUniformityAndTypes[i], MUI, MRI))
|
|
return false;
|
|
}
|
|
|
|
// More complex check.
|
|
if (TestFunc)
|
|
return TestFunc(MI);
|
|
|
|
return true;
|
|
}
|
|
|
|
SetOfRulesForOpcode::SetOfRulesForOpcode() = default;
|
|
|
|
SetOfRulesForOpcode::SetOfRulesForOpcode(FastRulesTypes FastTypes)
|
|
: FastTypes(FastTypes) {}
|
|
|
|
UniformityLLTOpPredicateID LLTToId(LLT Ty) {
|
|
if (Ty == LLT::scalar(16))
|
|
return S16;
|
|
if (Ty == LLT::scalar(32))
|
|
return S32;
|
|
if (Ty == LLT::scalar(64))
|
|
return S64;
|
|
if (Ty == LLT::fixed_vector(2, 16))
|
|
return V2S16;
|
|
if (Ty == LLT::fixed_vector(2, 32))
|
|
return V2S32;
|
|
if (Ty == LLT::fixed_vector(3, 32))
|
|
return V3S32;
|
|
if (Ty == LLT::fixed_vector(4, 32))
|
|
return V4S32;
|
|
return _;
|
|
}
|
|
|
|
UniformityLLTOpPredicateID LLTToBId(LLT Ty) {
|
|
if (Ty == LLT::scalar(32) || Ty == LLT::fixed_vector(2, 16) ||
|
|
isAnyPtr(Ty, 32))
|
|
return B32;
|
|
if (Ty == LLT::scalar(64) || Ty == LLT::fixed_vector(2, 32) ||
|
|
Ty == LLT::fixed_vector(4, 16) || isAnyPtr(Ty, 64))
|
|
return B64;
|
|
if (Ty == LLT::fixed_vector(3, 32))
|
|
return B96;
|
|
if (Ty == LLT::fixed_vector(4, 32) || Ty == LLT::fixed_vector(2, 64) ||
|
|
Ty == LLT::fixed_vector(8, 16) || isAnyPtr(Ty, 128))
|
|
return B128;
|
|
return _;
|
|
}
|
|
|
|
const RegBankLLTMapping *
|
|
SetOfRulesForOpcode::findMappingForMI(const MachineInstr &MI,
|
|
const MachineRegisterInfo &MRI,
|
|
const MachineUniformityInfo &MUI) const {
|
|
// Search in "Fast Rules".
|
|
// Note: if fast rules are enabled, RegBankLLTMapping must be added in each
|
|
// slot that could "match fast Predicate". If not, InvalidMapping is
|
|
// returned which results in failure, does not search "Slow Rules".
|
|
if (FastTypes != NoFastRules) {
|
|
Register Reg = MI.getOperand(0).getReg();
|
|
int Slot;
|
|
if (FastTypes == StandardB)
|
|
Slot = getFastPredicateSlot(LLTToBId(MRI.getType(Reg)));
|
|
else
|
|
Slot = getFastPredicateSlot(LLTToId(MRI.getType(Reg)));
|
|
|
|
if (Slot != -1)
|
|
return MUI.isUniform(Reg) ? &Uni[Slot] : &Div[Slot];
|
|
}
|
|
|
|
// Slow search for more complex rules.
|
|
for (const RegBankLegalizeRule &Rule : Rules) {
|
|
if (Rule.Predicate.match(MI, MUI, MRI))
|
|
return &Rule.OperandMapping;
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
void SetOfRulesForOpcode::addRule(RegBankLegalizeRule Rule) {
|
|
Rules.push_back(Rule);
|
|
}
|
|
|
|
void SetOfRulesForOpcode::addFastRuleDivergent(UniformityLLTOpPredicateID Ty,
|
|
RegBankLLTMapping RuleApplyIDs) {
|
|
int Slot = getFastPredicateSlot(Ty);
|
|
assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");
|
|
Div[Slot] = std::move(RuleApplyIDs);
|
|
}
|
|
|
|
void SetOfRulesForOpcode::addFastRuleUniform(UniformityLLTOpPredicateID Ty,
|
|
RegBankLLTMapping RuleApplyIDs) {
|
|
int Slot = getFastPredicateSlot(Ty);
|
|
assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");
|
|
Uni[Slot] = std::move(RuleApplyIDs);
|
|
}
|
|
|
|
int SetOfRulesForOpcode::getFastPredicateSlot(
|
|
UniformityLLTOpPredicateID Ty) const {
|
|
switch (FastTypes) {
|
|
case Standard: {
|
|
switch (Ty) {
|
|
case S32:
|
|
return 0;
|
|
case S16:
|
|
return 1;
|
|
case S64:
|
|
return 2;
|
|
case V2S16:
|
|
return 3;
|
|
default:
|
|
return -1;
|
|
}
|
|
}
|
|
case StandardB: {
|
|
switch (Ty) {
|
|
case B32:
|
|
return 0;
|
|
case B64:
|
|
return 1;
|
|
case B96:
|
|
return 2;
|
|
case B128:
|
|
return 3;
|
|
default:
|
|
return -1;
|
|
}
|
|
}
|
|
case Vector: {
|
|
switch (Ty) {
|
|
case S32:
|
|
return 0;
|
|
case V2S32:
|
|
return 1;
|
|
case V3S32:
|
|
return 2;
|
|
case V4S32:
|
|
return 3;
|
|
default:
|
|
return -1;
|
|
}
|
|
}
|
|
default:
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
RegBankLegalizeRules::RuleSetInitializer
|
|
RegBankLegalizeRules::addRulesForGOpcs(std::initializer_list<unsigned> OpcList,
|
|
FastRulesTypes FastTypes) {
|
|
return RuleSetInitializer(OpcList, GRulesAlias, GRules, FastTypes);
|
|
}
|
|
|
|
RegBankLegalizeRules::RuleSetInitializer
|
|
RegBankLegalizeRules::addRulesForIOpcs(std::initializer_list<unsigned> OpcList,
|
|
FastRulesTypes FastTypes) {
|
|
return RuleSetInitializer(OpcList, IRulesAlias, IRules, FastTypes);
|
|
}
|
|
|
|
const SetOfRulesForOpcode *
|
|
RegBankLegalizeRules::getRulesForOpc(MachineInstr &MI) const {
|
|
unsigned Opc = MI.getOpcode();
|
|
if (Opc == AMDGPU::G_INTRINSIC || Opc == AMDGPU::G_INTRINSIC_CONVERGENT ||
|
|
Opc == AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS ||
|
|
Opc == AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
|
|
unsigned IntrID = cast<GIntrinsic>(MI).getIntrinsicID();
|
|
auto IRAIt = IRulesAlias.find(IntrID);
|
|
if (IRAIt == IRulesAlias.end())
|
|
return nullptr;
|
|
return &IRules.at(IRAIt->second);
|
|
}
|
|
|
|
auto GRAIt = GRulesAlias.find(Opc);
|
|
if (GRAIt == GRulesAlias.end())
|
|
return nullptr;
|
|
return &GRules.at(GRAIt->second);
|
|
}
|
|
|
|
// Syntactic sugar wrapper for predicate lambda that enables '&&', '||' and '!'.
|
|
class Predicate {
|
|
private:
|
|
struct Elt {
|
|
// Save formula composed of Pred, '&&', '||' and '!' as a jump table.
|
|
// Sink ! to Pred. For example !((A && !B) || C) -> (!A || B) && !C
|
|
// Sequences of && and || will be represented by jumps, for example:
|
|
// (A && B && ... X) or (A && B && ... X) || Y
|
|
// A == true jump to B
|
|
// A == false jump to end or Y, result is A(false) or Y
|
|
// (A || B || ... X) or (A || B || ... X) && Y
|
|
// A == true jump to end or Y, result is A(true) or Y
|
|
// A == false jump to B
|
|
// Notice that when negating expression, we simply flip Neg on each Pred
|
|
// and swap TJumpOffset and FJumpOffset (&& becomes ||, || becomes &&).
|
|
std::function<bool(const MachineInstr &)> Pred;
|
|
bool Neg; // Neg of Pred is calculated before jump
|
|
unsigned TJumpOffset;
|
|
unsigned FJumpOffset;
|
|
};
|
|
|
|
SmallVector<Elt, 8> Expression;
|
|
|
|
Predicate(SmallVectorImpl<Elt> &&Expr) { Expression.swap(Expr); };
|
|
|
|
public:
|
|
Predicate(std::function<bool(const MachineInstr &)> Pred) {
|
|
Expression.push_back({Pred, false, 1, 1});
|
|
};
|
|
|
|
bool operator()(const MachineInstr &MI) const {
|
|
unsigned Idx = 0;
|
|
unsigned ResultIdx = Expression.size();
|
|
bool Result;
|
|
do {
|
|
Result = Expression[Idx].Pred(MI);
|
|
Result = Expression[Idx].Neg ? !Result : Result;
|
|
if (Result) {
|
|
Idx += Expression[Idx].TJumpOffset;
|
|
} else {
|
|
Idx += Expression[Idx].FJumpOffset;
|
|
}
|
|
} while ((Idx != ResultIdx));
|
|
|
|
return Result;
|
|
};
|
|
|
|
Predicate operator!() const {
|
|
SmallVector<Elt, 8> NegExpression;
|
|
for (const Elt &ExprElt : Expression) {
|
|
NegExpression.push_back({ExprElt.Pred, !ExprElt.Neg, ExprElt.FJumpOffset,
|
|
ExprElt.TJumpOffset});
|
|
}
|
|
return Predicate(std::move(NegExpression));
|
|
};
|
|
|
|
Predicate operator&&(const Predicate &RHS) const {
|
|
SmallVector<Elt, 8> AndExpression = Expression;
|
|
|
|
unsigned RHSSize = RHS.Expression.size();
|
|
unsigned ResultIdx = Expression.size();
|
|
for (unsigned i = 0; i < ResultIdx; ++i) {
|
|
// LHS results in false, whole expression results in false.
|
|
if (i + AndExpression[i].FJumpOffset == ResultIdx)
|
|
AndExpression[i].FJumpOffset += RHSSize;
|
|
}
|
|
|
|
AndExpression.append(RHS.Expression);
|
|
|
|
return Predicate(std::move(AndExpression));
|
|
}
|
|
|
|
Predicate operator||(const Predicate &RHS) const {
|
|
SmallVector<Elt, 8> OrExpression = Expression;
|
|
|
|
unsigned RHSSize = RHS.Expression.size();
|
|
unsigned ResultIdx = Expression.size();
|
|
for (unsigned i = 0; i < ResultIdx; ++i) {
|
|
// LHS results in true, whole expression results in true.
|
|
if (i + OrExpression[i].TJumpOffset == ResultIdx)
|
|
OrExpression[i].TJumpOffset += RHSSize;
|
|
}
|
|
|
|
OrExpression.append(RHS.Expression);
|
|
|
|
return Predicate(std::move(OrExpression));
|
|
}
|
|
};
|
|
|
|
// Initialize rules
|
|
RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
|
|
MachineRegisterInfo &_MRI)
|
|
: ST(&_ST), MRI(&_MRI) {
|
|
|
|
addRulesForGOpcs({G_ADD, G_SUB}, Standard)
|
|
.Uni(S16, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackAExt})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr64}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr64}});
|
|
|
|
addRulesForGOpcs({G_UADDO, G_USUBO}, Standard)
|
|
.Uni(S32, {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32}})
|
|
.Div(S32, {{Vgpr32, Vcc}, {Vgpr32, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_UADDE, G_USUBE, G_SADDE, G_SSUBE}, Standard)
|
|
.Uni(S32, {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr32AExtBoolInReg}})
|
|
.Div(S32, {{Vgpr32, Vcc}, {Vgpr32, Vgpr32, Vcc}});
|
|
|
|
addRulesForGOpcs({G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT}, Standard)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16, Vgpr16}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
|
|
|
|
bool HasVecMulU64 = ST->hasVectorMulU64();
|
|
addRulesForGOpcs({G_MUL}, Standard)
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(S64, {{SgprB64}, {SgprB64, SgprB64}})
|
|
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
|
|
.Uni(S16, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
|
|
.Div(S64, {{VgprB64}, {VgprB64, VgprB64}}, HasVecMulU64)
|
|
.Div(S64, {{VgprB64}, {VgprB64, VgprB64}, SplitTo32Mul}, !HasVecMulU64);
|
|
|
|
bool hasMulHi = ST->hasScalarMulHiInsts();
|
|
addRulesForGOpcs({G_UMULH, G_SMULH}, Standard)
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}}, hasMulHi)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}}, !hasMulHi);
|
|
|
|
addRulesForGOpcs({G_AMDGPU_MAD_U64_U32}, Standard)
|
|
.Div(S64, {{Vgpr64, Vcc}, {Vgpr32, Vgpr32, Vgpr64}})
|
|
.Uni(S64, {{Sgpr64, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr64}, UniMAD64});
|
|
|
|
bool HasScalarSMulU64 = ST->hasScalarSMulU64();
|
|
addRulesForGOpcs({G_AMDGPU_S_MUL_U64_U32, G_AMDGPU_S_MUL_I64_I32}, Standard)
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr64}, UniMul64}, HasScalarSMulU64)
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr64}, DivSMulToMAD});
|
|
|
|
addRulesForGOpcs({G_XOR, G_OR, G_AND}, StandardB)
|
|
.Any({{UniS1}, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}}})
|
|
.Any({{DivS1}, {{Vcc}, {Vcc, Vcc}}})
|
|
.Any({{UniS16}, {{Sgpr16}, {Sgpr16, Sgpr16}}})
|
|
.Any({{DivS16}, {{Vgpr16}, {Vgpr16, Vgpr16}}})
|
|
.Uni(B32, {{SgprB32}, {SgprB32, SgprB32}})
|
|
.Div(B32, {{VgprB32}, {VgprB32, VgprB32}})
|
|
.Uni(B64, {{SgprB64}, {SgprB64, SgprB64}})
|
|
.Div(B64, {{VgprB64}, {VgprB64, VgprB64}, SplitTo32});
|
|
|
|
addRulesForGOpcs({G_SHL}, Standard)
|
|
.Uni(S16, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32ZExt}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_LSHR}, Standard)
|
|
.Uni(S16, {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_ASHR}, Standard)
|
|
.Uni(S16, {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32ZExt}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_FSHR}, Standard)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_FRAME_INDEX}).Any({{UniP5, _}, {{SgprP5}, {None}}});
|
|
|
|
addRulesForGOpcs({G_UBFX, G_SBFX}, Standard)
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}, S_BFE})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32, Sgpr32}, S_BFE})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32, Vgpr32}, V_BFE});
|
|
|
|
addRulesForGOpcs({G_SMIN, G_SMAX}, Standard)
|
|
.Uni(S16, {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32SExt}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
|
|
|
|
addRulesForGOpcs({G_UMIN, G_UMAX}, Standard)
|
|
.Uni(S16, {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
|
|
|
|
// Note: we only write S1 rules for G_IMPLICIT_DEF, G_CONSTANT and G_FCONSTANT
|
|
// here, rest is trivially regbankselected earlier
|
|
addRulesForGOpcs({G_IMPLICIT_DEF}).Any({{UniS1}, {{Sgpr32Trunc}, {}}});
|
|
addRulesForGOpcs({G_CONSTANT})
|
|
.Any({{UniS1, _}, {{Sgpr32Trunc}, {None}, UniCstExt}});
|
|
|
|
addRulesForGOpcs({G_FREEZE})
|
|
.Any({{UniS1}, {{Sgpr32Trunc}, {Sgpr32AExt}}})
|
|
.Any({{DivS1}, {{Vcc}, {Vcc}}})
|
|
.Any({{UniS16}, {{Sgpr16}, {Sgpr16}}})
|
|
.Any({{UniBRC}, {{SgprBRC}, {SgprBRC}}})
|
|
.Any({{DivBRC}, {{VgprBRC}, {VgprBRC}}});
|
|
|
|
addRulesForGOpcs({G_UNMERGE_VALUES})
|
|
.Any({{UniS16}, {{}, {}, UnmergeToShiftTrunc}})
|
|
.Any({{UniBRC}, {{}, {}, VerifyAllSgpr}})
|
|
.Any({{DivBRC}, {{}, {}, ApplyAllVgpr}});
|
|
|
|
// LOAD {Div}, {{VgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
|
|
// LOAD {Uni}, {{UniInVgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
|
|
// LOAD_NORET {}, {{}, {Imm, VgprSrc, ..., Sgpr_WF_RsrcIdx}}
|
|
// STORE {}, {{}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
|
|
addRulesForGOpcs({G_AMDGPU_INTRIN_IMAGE_LOAD, G_AMDGPU_INTRIN_IMAGE_LOAD_D16,
|
|
G_AMDGPU_INTRIN_IMAGE_LOAD_NORET,
|
|
G_AMDGPU_INTRIN_IMAGE_STORE,
|
|
G_AMDGPU_INTRIN_IMAGE_STORE_D16})
|
|
.Any({{}, {{}, {}, ApplyINTRIN_IMAGE}});
|
|
|
|
Predicate isSignedICmp([](const MachineInstr &MI) -> bool {
|
|
auto Pred =
|
|
static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
|
|
return CmpInst::isSigned(Pred);
|
|
});
|
|
|
|
Predicate isEqualityICmp([](const MachineInstr &MI) -> bool {
|
|
auto Pred =
|
|
static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
|
|
return ICmpInst::isEquality(Pred);
|
|
});
|
|
|
|
bool HasScalarCompareEq64 = ST->hasScalarCompareEq64();
|
|
// clang-format off
|
|
addRulesForGOpcs({G_ICMP})
|
|
.Any({{{UniS1, _, S16}, isEqualityICmp}, {{Sgpr32Trunc}, {None, Sgpr32ZExt, Sgpr32ZExt}}})
|
|
.Any({{{UniS1, _, S16}, !isEqualityICmp && isSignedICmp}, {{Sgpr32Trunc}, {None, Sgpr32SExt, Sgpr32SExt}}})
|
|
.Any({{{UniS1, _, S16}, !isEqualityICmp && !isSignedICmp}, {{Sgpr32Trunc}, {None, Sgpr32ZExt, Sgpr32ZExt}}})
|
|
.Any({{{DivS1, _, S16}}, {{Vcc}, {None, Vgpr16, Vgpr16}}})
|
|
.Any({{{UniS1, _, S32}}, {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}})
|
|
.Any({{{DivS1, _, S32}}, {{Vcc}, {None, Vgpr32, Vgpr32}}})
|
|
.Any({{{UniS1, _, S64}, isEqualityICmp}, {{Sgpr32Trunc}, {None, Sgpr64, Sgpr64}}}, HasScalarCompareEq64)
|
|
.Any({{{UniS1, _, S64}, isEqualityICmp}, {{UniInVcc}, {None, Vgpr64, Vgpr64}}}, !HasScalarCompareEq64)
|
|
.Any({{{UniS1, _, S64}, !isEqualityICmp}, {{UniInVcc}, {None, Vgpr64, Vgpr64}}})
|
|
.Any({{{DivS1, _, S64}}, {{Vcc}, {None, Vgpr64, Vgpr64}}})
|
|
.Any({{{UniS1, _, Ptr32}}, {{Sgpr32Trunc}, {None, SgprPtr32, SgprPtr32}}})
|
|
.Any({{{DivS1, _, Ptr32}}, {{Vcc}, {None, VgprPtr32, VgprPtr32}}})
|
|
.Any({{{UniS1, _, Ptr64}, isEqualityICmp}, {{Sgpr32Trunc}, {None, SgprPtr64, SgprPtr64}}}, HasScalarCompareEq64)
|
|
.Any({{{UniS1, _, Ptr64}, isEqualityICmp}, {{UniInVcc}, {None, VgprPtr64, VgprPtr64}}}, !HasScalarCompareEq64)
|
|
.Any({{{UniS1, _, Ptr64}, !isEqualityICmp}, {{UniInVcc}, {None, VgprPtr64, VgprPtr64}}})
|
|
.Any({{{DivS1, _, Ptr64}}, {{Vcc}, {None, VgprPtr64, VgprPtr64}}});
|
|
// clang-format on
|
|
|
|
addRulesForGOpcs({G_BRCOND})
|
|
.Any({{UniS1}, {{}, {Sgpr32AExtBoolInReg}}})
|
|
.Any({{DivS1}, {{}, {Vcc}}});
|
|
|
|
addRulesForGOpcs({G_BR}).Any({{_}, {{}, {None}}});
|
|
|
|
addRulesForGOpcs({G_SELECT}, StandardB)
|
|
.Any({{DivS16}, {{Vgpr16}, {Vcc, Vgpr16, Vgpr16}}})
|
|
.Any({{UniS16}, {{Sgpr16}, {Sgpr32AExtBoolInReg, Sgpr16, Sgpr16}}})
|
|
.Div(B32, {{VgprB32}, {Vcc, VgprB32, VgprB32}})
|
|
.Uni(B32, {{SgprB32}, {Sgpr32AExtBoolInReg, SgprB32, SgprB32}})
|
|
.Div(B64, {{VgprB64}, {Vcc, VgprB64, VgprB64}, SplitTo32Select})
|
|
.Uni(B64, {{SgprB64}, {Sgpr32AExtBoolInReg, SgprB64, SgprB64}});
|
|
|
|
addRulesForGOpcs({G_ANYEXT})
|
|
.Any({{UniS16, S1}, {{None}, {None}}}) // should be combined away
|
|
.Any({{UniS32, S1}, {{None}, {None}}}) // should be combined away
|
|
.Any({{UniS64, S1}, {{None}, {None}}}) // should be combined away
|
|
.Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})
|
|
.Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})
|
|
.Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})
|
|
.Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})
|
|
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})
|
|
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})
|
|
.Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});
|
|
|
|
bool Has16bitCmp = ST->has16BitInsts();
|
|
|
|
// In global-isel G_TRUNC in-reg is treated as no-op, inst selected into COPY.
|
|
// It is up to user to deal with truncated bits.
|
|
addRulesForGOpcs({G_TRUNC})
|
|
.Any({{UniS1, UniS16}, {{None}, {None}}}) // should be combined away
|
|
.Any({{UniS1, UniS32}, {{None}, {None}}}) // should be combined away
|
|
.Any({{UniS1, UniS64}, {{None}, {None}}}) // should be combined away
|
|
.Any({{UniS16, S32}, {{Sgpr16}, {Sgpr32}}})
|
|
.Any({{DivS16, S32}, {{Vgpr16}, {Vgpr32}}})
|
|
.Any({{UniS32, S64}, {{Sgpr32}, {Sgpr64}}})
|
|
.Any({{DivS32, S64}, {{Vgpr32}, {Vgpr64}}})
|
|
.Any({{UniV2S16, V2S32}, {{SgprV2S16}, {SgprV2S32}}})
|
|
.Any({{DivV2S16, V2S32}, {{VgprV2S16}, {VgprV2S32}}})
|
|
// This is non-trivial. VgprToVccCopy is done using compare instruction.
|
|
.Any({{DivS1, DivS16}, {{Vcc}, {Vgpr16}, VgprToVccCopy}}, Has16bitCmp)
|
|
.Any({{DivS1, DivS16}, {{Vcc}, {Vgpr32AExt}, VgprToVccCopy}},
|
|
!Has16bitCmp)
|
|
.Any({{DivS1, DivS32}, {{Vcc}, {Vgpr32}, VgprToVccCopy}})
|
|
.Any({{DivS1, DivS64}, {{Vcc}, {Vgpr64}, VgprToVccCopy}});
|
|
|
|
addRulesForGOpcs({G_ZEXT})
|
|
.Any({{UniS16, S1}, {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
|
|
.Any({{UniS32, S1}, {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
|
|
.Any({{UniS64, S1}, {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
|
|
.Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})
|
|
.Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})
|
|
.Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})
|
|
.Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})
|
|
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})
|
|
// not extending S16 to S32 is questionable.
|
|
.Any({{UniS64, S16}, {{Sgpr64}, {Sgpr32ZExt}, Ext32To64}})
|
|
.Any({{DivS64, S16}, {{Vgpr64}, {Vgpr32ZExt}, Ext32To64}})
|
|
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})
|
|
.Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});
|
|
|
|
addRulesForGOpcs({G_SEXT})
|
|
.Any({{UniS16, S1}, {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
|
|
.Any({{UniS32, S1}, {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
|
|
.Any({{UniS64, S1}, {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
|
|
.Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})
|
|
.Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})
|
|
.Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})
|
|
.Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})
|
|
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})
|
|
// not extending S16 to S32 is questionable.
|
|
.Any({{UniS64, S16}, {{Sgpr64}, {Sgpr32SExt}, Ext32To64}})
|
|
.Any({{DivS64, S16}, {{Vgpr64}, {Vgpr32SExt}, Ext32To64}})
|
|
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})
|
|
.Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});
|
|
|
|
addRulesForGOpcs({G_SEXT_INREG})
|
|
.Any({{UniS32, S32}, {{Sgpr32}, {Sgpr32}}})
|
|
.Any({{DivS32, S32}, {{Vgpr32}, {Vgpr32}}})
|
|
.Any({{UniS64, S64}, {{Sgpr64}, {Sgpr64}}})
|
|
.Any({{DivS64, S64}, {{Vgpr64}, {Vgpr64}, SplitTo32SExtInReg}});
|
|
|
|
addRulesForGOpcs({G_ASSERT_ZEXT, G_ASSERT_SEXT}, Standard)
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Imm}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Imm}})
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64, Imm}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Imm}});
|
|
|
|
addRulesForGOpcs({G_ASSERT_ALIGN}, Standard)
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32}})
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64}})
|
|
.Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32}}})
|
|
.Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32}}})
|
|
.Any({{UniPtr64}, {{SgprPtr64}, {SgprPtr64}}})
|
|
.Any({{DivPtr64}, {{VgprPtr64}, {VgprPtr64}}});
|
|
|
|
// Atomic read-modify-write operations: result and value are always VGPR,
|
|
// pointer varies by address space.
|
|
addRulesForGOpcs({G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_XCHG,
|
|
G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
|
|
G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN,
|
|
G_ATOMICRMW_UMAX, G_ATOMICRMW_UINC_WRAP,
|
|
G_ATOMICRMW_UDEC_WRAP})
|
|
.Any({{DivS32, P0, S32}, {{Vgpr32}, {VgprP0, Vgpr32}}})
|
|
.Any({{DivS64, P0, S64}, {{Vgpr64}, {VgprP0, Vgpr64}}})
|
|
.Any({{DivS32, P1, S32}, {{Vgpr32}, {VgprP1, Vgpr32}}})
|
|
.Any({{DivS64, P1, S64}, {{Vgpr64}, {VgprP1, Vgpr64}}})
|
|
.Any({{DivS32, P3, S32}, {{Vgpr32}, {VgprP3, Vgpr32}}})
|
|
.Any({{DivS64, P3, S64}, {{Vgpr64}, {VgprP3, Vgpr64}}});
|
|
|
|
bool HasAtomicFlatPkAdd16Insts = ST->hasAtomicFlatPkAdd16Insts();
|
|
bool HasAtomicBufferGlobalPkAddF16Insts =
|
|
ST->hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
|
|
ST->hasAtomicBufferGlobalPkAddF16Insts();
|
|
bool HasAtomicDsPkAdd16Insts = ST->hasAtomicDsPkAdd16Insts();
|
|
addRulesForGOpcs({G_ATOMICRMW_FADD})
|
|
.Any({{DivS32, P0, S32}, {{Vgpr32}, {VgprP0, Vgpr32}}})
|
|
.Any({{DivS64, P0, S64}, {{Vgpr64}, {VgprP0, Vgpr64}}})
|
|
.Any({{DivS32, P1, S32}, {{Vgpr32}, {VgprP1, Vgpr32}}})
|
|
.Any({{DivS64, P1, S64}, {{Vgpr64}, {VgprP1, Vgpr64}}})
|
|
.Any({{DivS32, P3, S32}, {{Vgpr32}, {VgprP3, Vgpr32}}})
|
|
.Any({{DivS64, P3, S64}, {{Vgpr64}, {VgprP3, Vgpr64}}})
|
|
.Any({{DivV2S16, P0, V2S16}, {{VgprV2S16}, {VgprP0, VgprV2S16}}},
|
|
HasAtomicFlatPkAdd16Insts)
|
|
.Any({{DivV2S16, P1, V2S16}, {{VgprV2S16}, {VgprP1, VgprV2S16}}},
|
|
HasAtomicBufferGlobalPkAddF16Insts)
|
|
.Any({{DivV2S16, P3, V2S16}, {{VgprV2S16}, {VgprP3, VgprV2S16}}},
|
|
HasAtomicDsPkAdd16Insts);
|
|
|
|
addRulesForGOpcs({G_ATOMIC_CMPXCHG})
|
|
.Any({{DivS32, P2}, {{Vgpr32}, {VgprP2, Vgpr32, Vgpr32}}})
|
|
.Any({{DivS64, P2}, {{Vgpr64}, {VgprP2, Vgpr64, Vgpr64}}})
|
|
.Any({{DivS32, P3}, {{Vgpr32}, {VgprP3, Vgpr32, Vgpr32}}})
|
|
.Any({{DivS64, P3}, {{Vgpr64}, {VgprP3, Vgpr64, Vgpr64}}});
|
|
|
|
addRulesForGOpcs({G_AMDGPU_ATOMIC_CMPXCHG})
|
|
.Any({{DivS32, P0}, {{Vgpr32}, {VgprP0, VgprV2S32}}})
|
|
.Any({{DivS32, P1}, {{Vgpr32}, {VgprP1, VgprV2S32}}})
|
|
.Any({{DivS64, P0}, {{Vgpr64}, {VgprP0, VgprV2S64}}})
|
|
.Any({{DivS64, P1}, {{Vgpr64}, {VgprP1, VgprV2S64}}});
|
|
|
|
addRulesForGOpcs({G_AMDGPU_BUFFER_ATOMIC_CMPSWAP}, Standard)
|
|
.Div(S32, {{Vgpr32},
|
|
{Vgpr32, Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(S64, {{Vgpr64},
|
|
{Vgpr64, Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
|
|
|
|
addRulesForGOpcs({G_AMDGPU_BUFFER_ATOMIC_SWAP, G_AMDGPU_BUFFER_ATOMIC_UMAX,
|
|
G_AMDGPU_BUFFER_ATOMIC_UMIN, G_AMDGPU_BUFFER_ATOMIC_SMAX,
|
|
G_AMDGPU_BUFFER_ATOMIC_SMIN},
|
|
Standard)
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
|
|
|
|
bool hasSMRDx3 = ST->hasScalarDwordx3Loads();
|
|
bool hasSMRDSmall = ST->hasScalarSubwordLoads();
|
|
bool usesTrue16 = ST->useRealTrue16Insts();
|
|
|
|
Predicate isAlign16([](const MachineInstr &MI) -> bool {
|
|
return (*MI.memoperands_begin())->getAlign() >= Align(16);
|
|
});
|
|
|
|
Predicate isAlign4([](const MachineInstr &MI) -> bool {
|
|
return (*MI.memoperands_begin())->getAlign() >= Align(4);
|
|
});
|
|
|
|
Predicate isAtomicMMO([](const MachineInstr &MI) -> bool {
|
|
return (*MI.memoperands_begin())->isAtomic();
|
|
});
|
|
|
|
Predicate isUniMMO([](const MachineInstr &MI) -> bool {
|
|
return AMDGPU::isUniformMMO(*MI.memoperands_begin());
|
|
});
|
|
|
|
Predicate isConst([](const MachineInstr &MI) -> bool {
|
|
// Address space in MMO be different then address space on pointer.
|
|
const MachineMemOperand *MMO = *MI.memoperands_begin();
|
|
const unsigned AS = MMO->getAddrSpace();
|
|
return AS == AMDGPUAS::CONSTANT_ADDRESS ||
|
|
AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
|
|
});
|
|
|
|
Predicate isVolatileMMO([](const MachineInstr &MI) -> bool {
|
|
return (*MI.memoperands_begin())->isVolatile();
|
|
});
|
|
|
|
Predicate isInvMMO([](const MachineInstr &MI) -> bool {
|
|
return (*MI.memoperands_begin())->isInvariant();
|
|
});
|
|
|
|
Predicate isNoClobberMMO([](const MachineInstr &MI) -> bool {
|
|
return (*MI.memoperands_begin())->getFlags() & MONoClobber;
|
|
});
|
|
|
|
Predicate isNaturalAligned([](const MachineInstr &MI) -> bool {
|
|
const MachineMemOperand *MMO = *MI.memoperands_begin();
|
|
return MMO->getAlign() >= Align(MMO->getSize().getValue());
|
|
});
|
|
|
|
Predicate is8Or16BitMMO([](const MachineInstr &MI) -> bool {
|
|
const MachineMemOperand *MMO = *MI.memoperands_begin();
|
|
const unsigned MemSize = 8 * MMO->getSize().getValue();
|
|
return MemSize == 16 || MemSize == 8;
|
|
});
|
|
|
|
Predicate is32BitMMO([](const MachineInstr &MI) -> bool {
|
|
const MachineMemOperand *MMO = *MI.memoperands_begin();
|
|
return 8 * MMO->getSize().getValue() == 32;
|
|
});
|
|
|
|
auto isUL = !isAtomicMMO && isUniMMO && (isConst || !isVolatileMMO) &&
|
|
(isConst || isInvMMO || isNoClobberMMO);
|
|
|
|
// clang-format off
|
|
// TODO: S32Dst, 16-bit any-extending load should not appear on True16 targets
|
|
addRulesForGOpcs({G_LOAD})
|
|
// flat, addrspace(0), never uniform - flat_load
|
|
.Any({{DivS16, P0}, {{Vgpr16}, {VgprP0}}}, usesTrue16)
|
|
.Any({{DivB32, P0}, {{VgprB32}, {VgprP0}}}) // 32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{DivB64, P0}, {{VgprB64}, {VgprP0}}})
|
|
.Any({{DivB96, P0}, {{VgprB96}, {VgprP0}}})
|
|
.Any({{DivB128, P0}, {{VgprB128}, {VgprP0}}})
|
|
|
|
// global, addrspace(1)
|
|
// divergent - global_load
|
|
.Any({{DivS16, P1}, {{Vgpr16}, {VgprP1}}}, usesTrue16)
|
|
.Any({{DivB32, P1}, {{VgprB32}, {VgprP1}}}) //32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{DivB64, P1}, {{VgprB64}, {VgprP1}}})
|
|
.Any({{DivB96, P1}, {{VgprB96}, {VgprP1}}})
|
|
.Any({{DivB128, P1}, {{VgprB128}, {VgprP1}}})
|
|
.Any({{DivB256, P1}, {{VgprB256}, {VgprP1}, SplitLoad}})
|
|
.Any({{DivB512, P1}, {{VgprB512}, {VgprP1}, SplitLoad}})
|
|
|
|
// uniform - s_load
|
|
.Any({{{UniS16, P1}, isNaturalAligned && isUL}, {{Sgpr32Trunc}, {SgprP1}}}, usesTrue16 && hasSMRDSmall) // s16 load
|
|
.Any({{{UniS16, P1}, isAlign4 && isUL}, {{Sgpr32Trunc}, {SgprP1}, WidenMMOToS32}}, usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load
|
|
.Any({{{UniB32, P1}, isNaturalAligned && isUL}, {{SgprB32}, {SgprP1}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
|
|
// TODO: SplitLoad when !isNaturalAligned && isUL and target hasSMRDSmall
|
|
.Any({{{UniB32, P1}, is8Or16BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP1}, WidenMMOToS32}}, !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load
|
|
.Any({{{UniB32, P1}, is32BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP1}}}) //32-bit load
|
|
.Any({{{UniB64, P1}, isAlign4 && isUL}, {{SgprB64}, {SgprP1}}})
|
|
.Any({{{UniB96, P1}, isAlign16 && isUL}, {{SgprB96}, {SgprP1}, WidenLoad}}, !hasSMRDx3)
|
|
.Any({{{UniB96, P1}, isAlign4 && !isAlign16 && isUL}, {{SgprB96}, {SgprP1}, SplitLoad}}, !hasSMRDx3)
|
|
.Any({{{UniB96, P1}, isAlign4 && isUL}, {{SgprB96}, {SgprP1}}}, hasSMRDx3)
|
|
.Any({{{UniB128, P1}, isAlign4 && isUL}, {{SgprB128}, {SgprP1}}})
|
|
.Any({{{UniB256, P1}, isAlign4 && isUL}, {{SgprB256}, {SgprP1}}})
|
|
.Any({{{UniB512, P1}, isAlign4 && isUL}, {{SgprB512}, {SgprP1}}})
|
|
|
|
// Uniform via global or buffer load, for example volatile or non-aligned
|
|
// uniform load. Not using standard {{UniInVgprTy}, {VgprP1}} since it is
|
|
// selected as global_load, use SgprP1 for pointer instead to match
|
|
// patterns without flat-for-global, default for GFX7 and older.
|
|
// -> +flat-for-global + {{UniInVgprTy}, {SgprP1}} - global_load
|
|
// -> -flat-for-global + {{UniInVgprTy}, {SgprP1}} - buffer_load
|
|
.Any({{{UniS16, P1}, !isNaturalAligned || !isUL}, {{UniInVgprS16}, {SgprP1}}}, usesTrue16 && hasSMRDSmall) // s16 load
|
|
.Any({{{UniS16, P1}, !isAlign4 || !isUL}, {{UniInVgprS16}, {SgprP1}}}, usesTrue16 && !hasSMRDSmall) // s16 load
|
|
.Any({{{UniB32, P1}, !isNaturalAligned || !isUL}, {{UniInVgprB32}, {SgprP1}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{{UniB32, P1}, !isAlign4 || !isUL}, {{UniInVgprB32}, {SgprP1}}}, !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{{UniB64, P1}, !isAlign4 || !isUL}, {{UniInVgprB64}, {SgprP1}}})
|
|
.Any({{{UniB96, P1}, !isAlign4 || !isUL}, {{UniInVgprB96}, {SgprP1}}})
|
|
.Any({{{UniB128, P1}, !isAlign4 || !isUL}, {{UniInVgprB128}, {SgprP1}}})
|
|
.Any({{{UniB256, P1}, !isAlign4 || !isUL}, {{UniInVgprB256}, {SgprP1}, SplitLoad}})
|
|
.Any({{{UniB512, P1}, !isAlign4 || !isUL}, {{UniInVgprB512}, {SgprP1}, SplitLoad}})
|
|
|
|
// local, addrspace(3) - ds_load
|
|
.Any({{DivS16, P3}, {{Vgpr16}, {VgprP3}}}, usesTrue16)
|
|
.Any({{DivB32, P3}, {{VgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{DivB64, P3}, {{VgprB64}, {VgprP3}}})
|
|
.Any({{DivB96, P3}, {{VgprB96}, {VgprP3}}})
|
|
.Any({{DivB128, P3}, {{VgprB128}, {VgprP3}}})
|
|
|
|
.Any({{UniS16, P3}, {{UniInVgprS16}, {SgprP3}}}, usesTrue16) // 16-bit load
|
|
.Any({{UniB32, P3}, {{UniInVgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{UniB64, P3}, {{UniInVgprB64}, {VgprP3}}})
|
|
.Any({{UniB96, P3}, {{UniInVgprB96}, {VgprP3}}})
|
|
.Any({{UniB128, P3}, {{UniInVgprB128}, {VgprP3}}})
|
|
|
|
// constant, addrspace(4)
|
|
// divergent - global_load
|
|
.Any({{DivS16, P4}, {{Vgpr16}, {VgprP4}}}, usesTrue16)
|
|
.Any({{DivB32, P4}, {{VgprB32}, {VgprP4}}}) //32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{DivB64, P4}, {{VgprB64}, {VgprP4}}})
|
|
.Any({{DivB96, P4}, {{VgprB96}, {VgprP4}}})
|
|
.Any({{DivB128, P4}, {{VgprB128}, {VgprP4}}})
|
|
.Any({{DivB256, P4}, {{VgprB256}, {VgprP4}, SplitLoad}})
|
|
.Any({{DivB512, P4}, {{VgprB512}, {VgprP4}, SplitLoad}})
|
|
|
|
// uniform - s_load
|
|
.Any({{{UniS16, P4}, isNaturalAligned && isUL}, {{Sgpr32Trunc}, {SgprP4}}}, usesTrue16 && hasSMRDSmall) // s16 load
|
|
.Any({{{UniS16, P4}, isAlign4 && isUL}, {{Sgpr32Trunc}, {SgprP4}, WidenMMOToS32}}, usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load
|
|
.Any({{{UniB32, P4}, isNaturalAligned && isUL}, {{SgprB32}, {SgprP4}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{{UniB32, P4}, is8Or16BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP4}, WidenMMOToS32}}, !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load
|
|
.Any({{{UniB32, P4}, is32BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP4}}}) //32-bit load
|
|
.Any({{{UniB64, P4}, isAlign4 && isUL}, {{SgprB64}, {SgprP4}}})
|
|
.Any({{{UniB96, P4}, isAlign16 && isUL}, {{SgprB96}, {SgprP4}, WidenLoad}}, !hasSMRDx3)
|
|
.Any({{{UniB96, P4}, isAlign4 && !isAlign16 && isUL}, {{SgprB96}, {SgprP4}, SplitLoad}}, !hasSMRDx3)
|
|
.Any({{{UniB96, P4}, isAlign4 && isUL}, {{SgprB96}, {SgprP4}}}, hasSMRDx3)
|
|
.Any({{{UniB128, P4}, isAlign4 && isUL}, {{SgprB128}, {SgprP4}}})
|
|
.Any({{{UniB256, P4}, isAlign4 && isUL}, {{SgprB256}, {SgprP4}}})
|
|
.Any({{{UniB512, P4}, isAlign4 && isUL}, {{SgprB512}, {SgprP4}}})
|
|
|
|
// uniform in vgpr - global_load or buffer_load
|
|
.Any({{{UniS16, P4}, !isNaturalAligned || !isUL}, {{UniInVgprS16}, {SgprP4}}}, usesTrue16 && hasSMRDSmall) // s16 load
|
|
.Any({{{UniS16, P4}, !isAlign4 || !isUL}, {{UniInVgprS16}, {SgprP4}}}, usesTrue16 && !hasSMRDSmall) // s16 load
|
|
.Any({{{UniB32, P4}, !isNaturalAligned || !isUL}, {{UniInVgprB32}, {SgprP4}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{{UniB32, P4}, !isAlign4 || !isUL}, {{UniInVgprB32}, {SgprP4}}}, !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{{UniB64, P4}, !isAlign4 || !isUL}, {{UniInVgprB64}, {SgprP4}}})
|
|
.Any({{{UniB96, P4}, !isAlign4 || !isUL}, {{UniInVgprB96}, {SgprP4}}})
|
|
.Any({{{UniB128, P4}, !isAlign4 || !isUL}, {{UniInVgprB128}, {SgprP4}}})
|
|
.Any({{{UniB256, P4}, !isAlign4 || !isUL}, {{UniInVgprB256}, {SgprP4}, SplitLoad}})
|
|
.Any({{{UniB512, P4}, !isAlign4 || !isUL}, {{UniInVgprB512}, {SgprP4}, SplitLoad}})
|
|
|
|
// private, addrspace(5), never uniform - scratch_load
|
|
.Any({{DivS16, P5}, {{Vgpr16}, {VgprP5}}}, usesTrue16)
|
|
.Any({{DivB32, P5}, {{VgprB32}, {VgprP5}}}) // 32-bit load, 8-bit and 16-bit any-extending load
|
|
.Any({{DivB64, P5}, {{VgprB64}, {VgprP5}}})
|
|
.Any({{DivB96, P5}, {{VgprB96}, {VgprP5}}})
|
|
.Any({{DivB128, P5}, {{VgprB128}, {VgprP5}}})
|
|
|
|
.Any({{DivS32, Ptr128}, {{Vgpr32}, {VgprPtr128}}});
|
|
|
|
|
|
addRulesForGOpcs({G_ZEXTLOAD, G_SEXTLOAD}) // i8 and i16 zeroextending loads
|
|
.Any({{DivS32, P0}, {{Vgpr32}, {VgprP0}}})
|
|
|
|
.Any({{DivS32, P1}, {{Vgpr32}, {VgprP1}}})
|
|
.Any({{{UniS32, P1}, isAlign4 && isUL}, {{Sgpr32}, {SgprP1}, WidenMMOToS32}}, !hasSMRDSmall)
|
|
.Any({{{UniS32, P1}, isNaturalAligned && isUL}, {{Sgpr32}, {SgprP1}}}, hasSMRDSmall)
|
|
.Any({{{UniS32, P1}, !isAlign4 || !isUL}, {{UniInVgprS32}, {SgprP1}}}, !hasSMRDSmall)
|
|
.Any({{{UniS32, P1}, !isNaturalAligned || !isUL}, {{UniInVgprS32}, {SgprP1}}}, hasSMRDSmall)
|
|
|
|
.Any({{DivS32, P3}, {{Vgpr32}, {VgprP3}}})
|
|
.Any({{UniS32, P3}, {{UniInVgprS32}, {VgprP3}}})
|
|
|
|
.Any({{DivS32, P4}, {{Vgpr32}, {VgprP4}}})
|
|
.Any({{{UniS32, P4}, isAlign4 && isUL}, {{Sgpr32}, {SgprP4}, WidenMMOToS32}}, !hasSMRDSmall)
|
|
.Any({{{UniS32, P4}, isNaturalAligned && isUL}, {{Sgpr32}, {SgprP4}}}, hasSMRDSmall)
|
|
.Any({{{UniS32, P4}, !isAlign4 || !isUL}, {{UniInVgprS32}, {SgprP4}}}, !hasSMRDSmall)
|
|
.Any({{{UniS32, P4}, !isNaturalAligned || !isUL}, {{UniInVgprS32}, {SgprP4}}}, hasSMRDSmall)
|
|
|
|
.Any({{DivS32, P5}, {{Vgpr32}, {VgprP5}}});
|
|
|
|
addRulesForGOpcs({G_STORE})
|
|
// addrspace(0)
|
|
.Any({{S16, P0}, {{}, {Vgpr16, VgprP0}}}, usesTrue16) // 16-bit store
|
|
.Any({{B32, P0}, {{}, {VgprB32, VgprP0}}}) // 32-bit store, 8-bit and 16-bit truncating store
|
|
.Any({{B64, P0}, {{}, {VgprB64, VgprP0}}})
|
|
.Any({{B96, P0}, {{}, {VgprB96, VgprP0}}})
|
|
.Any({{B128, P0}, {{}, {VgprB128, VgprP0}}})
|
|
|
|
// addrspace(1), there are no stores to addrspace(4)
|
|
// For targets:
|
|
// - with "+flat-for-global" - global_store
|
|
// - without(-flat-for-global) - buffer_store addr64
|
|
.Any({{S16, DivP1}, {{}, {Vgpr16, VgprP1}}}, usesTrue16) // 16-bit store
|
|
.Any({{B32, DivP1}, {{}, {VgprB32, VgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store
|
|
.Any({{B64, DivP1}, {{}, {VgprB64, VgprP1}}})
|
|
.Any({{B96, DivP1}, {{}, {VgprB96, VgprP1}}})
|
|
.Any({{B128, DivP1}, {{}, {VgprB128, VgprP1}}})
|
|
|
|
// For UniP1, use sgpr ptr to match flat-for-global patterns. Targets:
|
|
// - with "+flat-for-global" - global_store for both sgpr and vgpr ptr
|
|
// - without(-flat-for-global) - need sgpr ptr to select buffer_store
|
|
.Any({{S16, UniP1}, {{}, {Vgpr16, SgprP1}}}, usesTrue16) // 16-bit store
|
|
.Any({{B32, UniP1}, {{}, {VgprB32, SgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store
|
|
.Any({{B64, UniP1}, {{}, {VgprB64, SgprP1}}})
|
|
.Any({{B96, UniP1}, {{}, {VgprB96, SgprP1}}})
|
|
.Any({{B128, UniP1}, {{}, {VgprB128, SgprP1}}})
|
|
|
|
// addrspace(3) and addrspace(5)
|
|
.Any({{S16, Ptr32}, {{}, {Vgpr16, VgprPtr32}}}, usesTrue16) // 16-bit store
|
|
.Any({{B32, Ptr32}, {{}, {VgprB32, VgprPtr32}}}) // 32-bit store, 8-bit and 16-bit truncating store
|
|
.Any({{B64, Ptr32}, {{}, {VgprB64, VgprPtr32}}})
|
|
.Any({{B96, Ptr32}, {{}, {VgprB96, VgprPtr32}}})
|
|
.Any({{B128, Ptr32}, {{}, {VgprB128, VgprPtr32}}});
|
|
|
|
// clang-format on
|
|
|
|
addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD, G_AMDGPU_BUFFER_LOAD_FORMAT,
|
|
G_AMDGPU_TBUFFER_LOAD_FORMAT},
|
|
StandardB)
|
|
.Div(B32, {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B32, {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(B96, {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B96, {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
|
|
|
|
addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD_USHORT, G_AMDGPU_BUFFER_LOAD_UBYTE,
|
|
G_AMDGPU_BUFFER_LOAD_SSHORT, G_AMDGPU_BUFFER_LOAD_SBYTE},
|
|
StandardB)
|
|
.Div(B32, {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B32, {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
|
|
|
|
addRulesForGOpcs(
|
|
{G_AMDGPU_BUFFER_LOAD_UBYTE_TFE, G_AMDGPU_BUFFER_LOAD_USHORT_TFE},
|
|
StandardB)
|
|
.Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
|
|
|
|
addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD_TFE, G_AMDGPU_BUFFER_LOAD_FORMAT_TFE},
|
|
StandardB)
|
|
.Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(B96, {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B96, {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Any({{DivB160}, {{VgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
|
|
.Any({{UniB160},
|
|
{{UniInVgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
|
|
|
|
addRulesForGOpcs(
|
|
{G_AMDGPU_BUFFER_LOAD_FORMAT_D16, G_AMDGPU_TBUFFER_LOAD_FORMAT_D16},
|
|
StandardB)
|
|
.Div(B32, {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B32, {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
|
|
.Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
|
|
|
|
addRulesForGOpcs({G_AMDGPU_BUFFER_STORE, G_AMDGPU_BUFFER_STORE_BYTE,
|
|
G_AMDGPU_BUFFER_STORE_SHORT, G_AMDGPU_BUFFER_STORE_FORMAT,
|
|
G_AMDGPU_BUFFER_STORE_FORMAT_D16,
|
|
G_AMDGPU_TBUFFER_STORE_FORMAT,
|
|
G_AMDGPU_TBUFFER_STORE_FORMAT_D16})
|
|
.Any({{B32}, {{}, {VgprB32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
|
|
.Any({{B64}, {{}, {VgprB64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
|
|
.Any({{B96}, {{}, {VgprB96, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
|
|
.Any({{B128}, {{}, {VgprB128, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
|
|
|
|
// Buffer atomics: resource descriptor + scalar offset are SGPR, data and
|
|
// address components are VGPR.
|
|
//
|
|
// Operand order (SIInstructions.td BufferAtomicGenericInstruction):
|
|
// dst = op vdata, rsrc, vindex, voffset, soffset, offset_imm, cachepolicy,
|
|
// idxen_imm
|
|
addRulesForGOpcs({G_AMDGPU_BUFFER_ATOMIC_FADD})
|
|
.Any({{S32, S32, V4S32, S32, S32, S32},
|
|
{{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
|
|
.Any({{S64, S64, V4S32, S32, S32, S32},
|
|
{{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
|
|
.Any({{V2S16, V2S16, V4S32, S32, S32, S32},
|
|
{{VgprV2S16},
|
|
{VgprV2S16, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
|
|
|
|
addRulesForGOpcs({G_PTR_ADD})
|
|
.Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
|
|
.Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
|
|
.Any({{UniPtr64}, {{SgprPtr64}, {SgprPtr64, Sgpr64}}})
|
|
.Any({{DivPtr64}, {{VgprPtr64}, {VgprPtr64, Vgpr64}}});
|
|
|
|
addRulesForGOpcs({G_INTTOPTR})
|
|
.Any({{UniPtr32}, {{SgprPtr32}, {Sgpr32}}})
|
|
.Any({{DivPtr32}, {{VgprPtr32}, {Vgpr32}}})
|
|
.Any({{UniPtr64}, {{SgprPtr64}, {Sgpr64}}})
|
|
.Any({{DivPtr64}, {{VgprPtr64}, {Vgpr64}}})
|
|
.Any({{UniPtr128}, {{SgprPtr128}, {Sgpr128}}})
|
|
.Any({{DivPtr128}, {{VgprPtr128}, {Vgpr128}}});
|
|
|
|
addRulesForGOpcs({G_PTRTOINT})
|
|
.Any({{UniS32}, {{Sgpr32}, {SgprPtr32}}})
|
|
.Any({{DivS32}, {{Vgpr32}, {VgprPtr32}}})
|
|
.Any({{UniS64}, {{Sgpr64}, {SgprPtr64}}})
|
|
.Any({{DivS64}, {{Vgpr64}, {VgprPtr64}}})
|
|
.Any({{UniS128}, {{Sgpr128}, {SgprPtr128}}})
|
|
.Any({{DivS128}, {{Vgpr128}, {VgprPtr128}}});
|
|
|
|
// FIXME: Update llvm/test/CodeGen/AMDGPU/ptrmask.ll to use GlobalISel.
|
|
// Currently crashes on P8 (buffer resource) tests due to legalizer issue.
|
|
addRulesForGOpcs({G_PTRMASK})
|
|
.Any({{UniP1}, {{SgprP1}, {SgprP1, Sgpr64}}})
|
|
.Any({{DivP1}, {{VgprP1}, {VgprP1, Vgpr64}}})
|
|
.Any({{UniP3}, {{SgprP3}, {SgprP3, Sgpr32}}})
|
|
.Any({{DivP3}, {{VgprP3}, {VgprP3, Vgpr32}}});
|
|
|
|
addRulesForGOpcs({G_ABS}, Standard).Uni(S16, {{Sgpr32Trunc}, {Sgpr32SExt}});
|
|
|
|
addRulesForGOpcs({G_BITREVERSE}, Standard)
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32}})
|
|
.Uni(S64, {{Sgpr64}, {Sgpr64}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64}});
|
|
|
|
addRulesForGOpcs({G_FENCE}).Any({{{}}, {{}, {}}});
|
|
|
|
addRulesForGOpcs({G_READSTEADYCOUNTER, G_READCYCLECOUNTER}, Standard)
|
|
.Uni(S64, {{Sgpr64}, {}});
|
|
|
|
addRulesForGOpcs({G_BLOCK_ADDR}).Any({{UniP0}, {{SgprP0}, {}}});
|
|
|
|
addRulesForGOpcs({G_GLOBAL_VALUE})
|
|
.Any({{UniP0}, {{SgprP0}, {}}})
|
|
.Any({{UniP1}, {{SgprP1}, {}}})
|
|
.Any({{UniP3}, {{SgprP3}, {}}})
|
|
.Any({{UniP4}, {{SgprP4}, {}}})
|
|
.Any({{UniP8}, {{SgprP8}, {}}});
|
|
|
|
addRulesForGOpcs({G_AMDGPU_WAVE_ADDRESS}).Any({{UniP5}, {{SgprP5}, {}}});
|
|
|
|
addRulesForGOpcs({G_SI_CALL})
|
|
.Any({{_, UniP0}, {{None}, {SgprP0}}})
|
|
.Any({{_, DivP0}, {{None}, {SgprP0Call_WF}}})
|
|
.Any({{_, UniP4}, {{None}, {SgprP4}}})
|
|
.Any({{_, DivP4}, {{None}, {SgprP4Call_WF}}});
|
|
|
|
bool hasSALUFloat = ST->hasSALUFloatInsts();
|
|
|
|
addRulesForGOpcs({G_FADD, G_FMUL, G_STRICT_FADD, G_STRICT_FMUL}, Standard)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16, Vgpr16}}, !hasSALUFloat)
|
|
.Uni(S16, {{Sgpr16}, {Sgpr16, Sgpr16}}, hasSALUFloat)
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}}, hasSALUFloat)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}}, !hasSALUFloat)
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(S64, {{UniInVgprS64}, {Vgpr64, Vgpr64}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr64}})
|
|
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}}, !hasSALUFloat)
|
|
.Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, ScalarizeToS16},
|
|
hasSALUFloat)
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
|
|
|
|
addRulesForGOpcs({G_FSUB, G_STRICT_FSUB}, Standard)
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(S16, {{Sgpr16}, {Sgpr16, Sgpr16}}, hasSALUFloat)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16, Vgpr16}}, !hasSALUFloat)
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}}, hasSALUFloat)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}}, !hasSALUFloat);
|
|
|
|
addRulesForGOpcs({G_FMAD}, Standard)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_FLDEXP, G_STRICT_FLDEXP}, Standard)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16, Vgpr16}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})
|
|
.Uni(S64, {{UniInVgprS64}, {Vgpr64, Vgpr32}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_FMA, G_STRICT_FMA}, Standard)
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})
|
|
.Uni(S64, {{UniInVgprS64}, {Vgpr64, Vgpr64, Vgpr64}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64, Vgpr64, Vgpr64}})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16, VgprV2S16}})
|
|
.Any({{UniV2S32}, {{UniInVgprV2S32}, {VgprV2S32, VgprV2S32, VgprV2S32}}})
|
|
.Any({{DivV2S32}, {{VgprV2S32}, {VgprV2S32, VgprV2S32, VgprV2S32}}})
|
|
.Uni(S16, {{Sgpr16}, {Sgpr16, Sgpr16, Sgpr16}}, hasSALUFloat)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}}, !hasSALUFloat)
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}}, hasSALUFloat)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}}, !hasSALUFloat)
|
|
.Uni(V2S16,
|
|
{{SgprV2S16}, {SgprV2S16, SgprV2S16, SgprV2S16}, ScalarizeToS16},
|
|
hasSALUFloat)
|
|
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16, VgprV2S16}},
|
|
!hasSALUFloat);
|
|
|
|
addRulesForGOpcs({G_AMDGPU_FMED3}, Standard)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
|
|
|
|
// TODO: This opcode is generated from the i64->i16 signed clamped pattern in
|
|
// the PreLegalizerCombiner. Move the combine to RegBankCombiner to keep more
|
|
// instructions on SALU.
|
|
addRulesForGOpcs({G_AMDGPU_SMED3}, Standard)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
|
|
|
|
// FNEG and FABS are either folded as source modifiers or can be selected as
|
|
// bitwise XOR and AND with Mask. XOR and AND are available on SALU but for
|
|
// targets without SALU float we still select them as VGPR since there would
|
|
// be no real sgpr use.
|
|
addRulesForGOpcs({G_FNEG, G_FABS}, Standard)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16}}, !hasSALUFloat)
|
|
.Uni(S16, {{Sgpr16}, {Sgpr16}}, hasSALUFloat)
|
|
.Div(S16, {{Vgpr16}, {Vgpr16}})
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32}}, !hasSALUFloat)
|
|
.Uni(S32, {{Sgpr32}, {Sgpr32}}, hasSALUFloat)
|
|
.Div(S32, {{Vgpr32}, {Vgpr32}})
|
|
.Uni(S64, {{UniInVgprS64}, {Vgpr64}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64}})
|
|
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16}}, !hasSALUFloat)
|
|
.Uni(V2S16, {{SgprV2S16}, {SgprV2S16}, ScalarizeToS16}, hasSALUFloat)
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16}})
|
|
.Any({{UniV2S32}, {{UniInVgprV2S32}, {VgprV2S32}}})
|
|
.Any({{DivV2S32}, {{VgprV2S32}, {VgprV2S32}}});
|
|
|
|
addRulesForGOpcs({G_FCANONICALIZE}, Standard)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32}})
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16}})
|
|
.Uni(S64, {{UniInVgprS64}, {Vgpr64}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64}})
|
|
.Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16}})
|
|
.Div(V2S16, {{VgprV2S16}, {VgprV2S16}})
|
|
.Any({{UniV2S32}, {{UniInVgprV2S32}, {VgprV2S32}}})
|
|
.Any({{DivV2S32}, {{VgprV2S32}, {VgprV2S32}}});
|
|
|
|
bool hasPST = ST->hasPseudoScalarTrans();
|
|
addRulesForGOpcs({G_FSQRT}, Standard)
|
|
.Div(S16, {{Vgpr16}, {Vgpr16}})
|
|
.Uni(S16, {{Sgpr16}, {Sgpr16}}, hasPST)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16}}, !hasPST);
|
|
|
|
addRulesForGOpcs({G_FPTOUI, G_FPTOSI})
|
|
.Any({{UniS16, S16}, {{UniInVgprS16}, {Vgpr16}}})
|
|
.Any({{DivS16, S16}, {{Vgpr16}, {Vgpr16}}})
|
|
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}}, hasSALUFloat)
|
|
.Any({{UniS32, S16}, {{UniInVgprS32}, {Vgpr16}}}, !hasSALUFloat)
|
|
.Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}})
|
|
.Any({{UniS32, S32}, {{Sgpr32}, {Sgpr32}}}, hasSALUFloat)
|
|
.Any({{UniS32, S32}, {{UniInVgprS32}, {Vgpr32}}}, !hasSALUFloat)
|
|
.Any({{DivS32, S32}, {{Vgpr32}, {Vgpr32}}})
|
|
.Any({{UniS32, S64}, {{UniInVgprS32}, {Vgpr64}}})
|
|
.Any({{DivS32, S64}, {{Vgpr32}, {Vgpr64}}});
|
|
|
|
addRulesForGOpcs({G_UITOFP, G_SITOFP})
|
|
.Any({{UniS16, S16}, {{UniInVgprS16}, {Vgpr16}}})
|
|
.Any({{DivS16, S16}, {{Vgpr16}, {Vgpr16}}})
|
|
.Any({{UniS16, S32}, {{Sgpr16}, {Sgpr32}}}, hasSALUFloat)
|
|
.Any({{UniS16, S32}, {{UniInVgprS16}, {Vgpr32}}}, !hasSALUFloat)
|
|
.Any({{DivS16, S32}, {{Vgpr16}, {Vgpr32}}})
|
|
.Any({{UniS32, S32}, {{Sgpr32}, {Sgpr32}}}, hasSALUFloat)
|
|
.Any({{UniS32, S32}, {{UniInVgprS32}, {Vgpr32}}}, !hasSALUFloat)
|
|
.Any({{DivS32, S32}, {{Vgpr32}, {Vgpr32}}})
|
|
.Any({{UniS64, S32}, {{UniInVgprS64}, {Vgpr32}}})
|
|
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}}});
|
|
|
|
addRulesForGOpcs({G_FPEXT})
|
|
.Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}})
|
|
.Any({{UniS64, S32}, {{UniInVgprS64}, {Vgpr32}}})
|
|
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}}})
|
|
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}}, hasSALUFloat)
|
|
.Any({{UniS32, S16}, {{UniInVgprS32}, {Vgpr16}}}, !hasSALUFloat);
|
|
|
|
addRulesForGOpcs({G_AMDGPU_CVT_PK_I16_I32}, Standard)
|
|
.Uni(V2S16, {{UniInVgprV2S16}, {Vgpr32, Vgpr32}})
|
|
.Div(V2S16, {{VgprV2S16}, {Vgpr32, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_AMDGPU_FMIN_LEGACY, G_AMDGPU_FMAX_LEGACY}, Standard)
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}});
|
|
|
|
addRulesForGOpcs({G_FPTRUNC})
|
|
.Any({{DivS16, S32}, {{Vgpr16}, {Vgpr32}}})
|
|
.Any({{UniS32, S64}, {{UniInVgprS32}, {Vgpr64}}})
|
|
.Any({{DivS32, S64}, {{Vgpr32}, {Vgpr64}}})
|
|
.Any({{UniV2S16, V2S32}, {{UniInVgprV2S16}, {VgprV2S32}}})
|
|
.Any({{DivV2S16, V2S32}, {{VgprV2S16}, {VgprV2S32}}})
|
|
.Any({{UniS16, S32}, {{Sgpr16}, {Sgpr32}}}, hasSALUFloat)
|
|
.Any({{UniS16, S32}, {{UniInVgprS16}, {Vgpr32}}}, !hasSALUFloat);
|
|
|
|
addRulesForGOpcs({G_IS_FPCLASS})
|
|
.Any({{DivS1, S16}, {{Vcc}, {Vgpr16}}})
|
|
.Any({{UniS1, S16}, {{UniInVcc}, {Vgpr16}}})
|
|
.Any({{DivS1, S32}, {{Vcc}, {Vgpr32}}})
|
|
.Any({{UniS1, S32}, {{UniInVcc}, {Vgpr32}}})
|
|
.Any({{DivS1, S64}, {{Vcc}, {Vgpr64}}})
|
|
.Any({{UniS1, S64}, {{UniInVcc}, {Vgpr64}}});
|
|
|
|
addRulesForGOpcs({G_FCMP}, Standard)
|
|
.Any({{UniS1, _, S16}, {{Sgpr32Trunc}, {None, Sgpr16, Sgpr16}}},
|
|
hasSALUFloat)
|
|
.Any({{UniS1, _, S16}, {{UniInVcc}, {None, Vgpr16, Vgpr16}}},
|
|
!hasSALUFloat)
|
|
.Any({{DivS1, _, S16}, {{Vcc}, {None, Vgpr16, Vgpr16}}})
|
|
.Any({{UniS1, _, S32}, {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}},
|
|
hasSALUFloat)
|
|
.Any({{UniS1, _, S32}, {{UniInVcc}, {None, Vgpr32, Vgpr32}}},
|
|
!hasSALUFloat)
|
|
.Any({{DivS1, _, S32}, {{Vcc}, {None, Vgpr32, Vgpr32}}})
|
|
.Any({{UniS1, _, S64}, {{UniInVcc}, {None, Vgpr64, Vgpr64}}})
|
|
.Any({{DivS1, _, S64}, {{Vcc}, {None, Vgpr64, Vgpr64}}});
|
|
|
|
addRulesForGOpcs({G_INTRINSIC_TRUNC, G_INTRINSIC_ROUNDEVEN, G_FFLOOR, G_FCEIL,
|
|
G_FEXP2, G_FLOG2},
|
|
Standard)
|
|
.Uni(S16, {{UniInVgprS16}, {Vgpr16}})
|
|
.Div(S16, {{Vgpr16}, {Vgpr16}})
|
|
.Uni(S32, {{UniInVgprS32}, {Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {Vgpr32}})
|
|
.Uni(S64, {{UniInVgprS64}, {Vgpr64}})
|
|
.Div(S64, {{Vgpr64}, {Vgpr64}});
|
|
|
|
using namespace Intrinsic;
|
|
|
|
addRulesForIOpcs({amdgcn_s_getpc}).Any({{UniS64, _}, {{Sgpr64}, {None}}});
|
|
|
|
addRulesForIOpcs({amdgcn_groupstaticsize}).Any({{S32}, {{Sgpr32}, {IntrId}}});
|
|
|
|
// This is "intrinsic lane mask" it was set to i32/i64 in llvm-ir.
|
|
addRulesForIOpcs({amdgcn_end_cf})
|
|
.Any({{_, UniS32}, {{}, {IntrId, Sgpr32}}})
|
|
.Any({{_, UniS64}, {{}, {IntrId, Sgpr64}}});
|
|
|
|
addRulesForIOpcs({amdgcn_if_break}, Standard)
|
|
.Uni(S64, {{Sgpr64}, {IntrId, Vcc, Sgpr64}})
|
|
.Uni(S32, {{Sgpr32}, {IntrId, Vcc, Sgpr32}});
|
|
|
|
addRulesForIOpcs({amdgcn_mbcnt_lo, amdgcn_mbcnt_hi}, Standard)
|
|
.Div(S32, {{}, {Vgpr32, None, Vgpr32, Vgpr32}});
|
|
|
|
addRulesForIOpcs({amdgcn_readfirstlane})
|
|
.Any({{UniS32, _, DivS32}, {{}, {Sgpr32, None, Vgpr32}}})
|
|
// this should not exist in the first place, it is from call lowering
|
|
// readfirstlaning just in case register is not in sgpr.
|
|
.Any({{UniS32, _, UniS32}, {{}, {Sgpr32, None, Vgpr32}}});
|
|
|
|
addRulesForIOpcs({amdgcn_s_sleep}).Any({{_, _}, {{}, {IntrId, Imm}}});
|
|
|
|
addRulesForIOpcs({amdgcn_bitop3}, Standard)
|
|
.Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
|
|
.Div(S16, {{Vgpr16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
|
|
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
|
|
|
|
addRulesForIOpcs({amdgcn_mul_u24, amdgcn_mul_i24}, Standard)
|
|
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
|
|
.Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr32, Vgpr32}})
|
|
.Div(S64, {{Vgpr64}, {IntrId, Vgpr32, Vgpr32}});
|
|
|
|
addRulesForIOpcs({amdgcn_mulhi_u24, amdgcn_mulhi_i24, amdgcn_fmul_legacy},
|
|
Standard)
|
|
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
|
|
|
|
addRulesForIOpcs({amdgcn_fma_legacy}, Standard)
|
|
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
|
|
|
|
addRulesForIOpcs({amdgcn_frexp_mant, amdgcn_fract}, Standard)
|
|
.Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}})
|
|
.Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
|
|
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}})
|
|
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
|
|
.Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64}})
|
|
.Div(S64, {{Vgpr64}, {IntrId, Vgpr64}});
|
|
|
|
addRulesForIOpcs({amdgcn_prng_b32})
|
|
.Any({{UniS32}, {{UniInVgprS32}, {IntrId, Vgpr32}}})
|
|
.Any({{DivS32}, {{Vgpr32}, {IntrId, Vgpr32}}});
|
|
|
|
addRulesForIOpcs({amdgcn_sffbh}, Standard)
|
|
.Uni(S32, {{Sgpr32}, {IntrId, Sgpr32}})
|
|
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32}});
|
|
|
|
addRulesForIOpcs({amdgcn_ubfe, amdgcn_sbfe}, Standard)
|
|
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
|
|
.Uni(S32, {{Sgpr32}, {IntrId, Sgpr32, Sgpr32, Sgpr32}, S_BFE})
|
|
.Uni(S64, {{Sgpr64}, {IntrId, Sgpr64, Sgpr32, Sgpr32}, S_BFE})
|
|
.Div(S64, {{Vgpr64}, {IntrId, Vgpr64, Vgpr32, Vgpr32}, V_BFE});
|
|
|
|
addRulesForIOpcs({amdgcn_global_load_tr_b64})
|
|
.Any({{DivB64}, {{VgprB64}, {IntrId, SgprP1}}})
|
|
.Any({{DivB32}, {{VgprB32}, {IntrId, SgprP1}}});
|
|
|
|
addRulesForIOpcs({amdgcn_global_load_tr_b128})
|
|
.Any({{DivB64}, {{VgprB64}, {IntrId, SgprP1}}})
|
|
.Any({{DivB128}, {{VgprB128}, {IntrId, SgprP1}}});
|
|
|
|
addRulesForIOpcs({amdgcn_global_atomic_ordered_add_b64})
|
|
.Any({{DivS64}, {{Vgpr64}, {IntrId, VgprP1, Vgpr64}}});
|
|
|
|
addRulesForIOpcs({amdgcn_raw_buffer_load_lds})
|
|
.Any({{_}, {{}, {IntrId, SgprV4S32, SgprP3, Imm, Vgpr32, Sgpr32}}});
|
|
|
|
addRulesForIOpcs({amdgcn_struct_buffer_load_lds})
|
|
.Any({{_},
|
|
{{}, {IntrId, SgprV4S32, SgprP3, Imm, Vgpr32, Vgpr32, Sgpr32}}});
|
|
|
|
addRulesForIOpcs({amdgcn_raw_ptr_buffer_load_lds})
|
|
.Any({{_}, {{}, {IntrId, SgprP8, SgprP3, Imm, Vgpr32, Sgpr32}}});
|
|
|
|
addRulesForIOpcs({amdgcn_struct_ptr_buffer_load_lds})
|
|
.Any({{_}, {{}, {IntrId, SgprP8, SgprP3, Imm, Vgpr32, Vgpr32, Sgpr32}}});
|
|
|
|
addRulesForIOpcs({amdgcn_wwm, amdgcn_strict_wwm}, StandardB)
|
|
.Div(B32, {{VgprB32}, {IntrId, VgprB32}})
|
|
.Uni(B32, {{SgprB32}, {IntrId, SgprB32}})
|
|
.Div(B64, {{VgprB64}, {IntrId, VgprB64}})
|
|
.Uni(B64, {{SgprB64}, {IntrId, SgprB64}})
|
|
.Div(B96, {{VgprB96}, {IntrId, VgprB96}})
|
|
.Uni(B96, {{SgprB96}, {IntrId, SgprB96}})
|
|
.Div(B128, {{VgprB128}, {IntrId, VgprB128}})
|
|
.Uni(B128, {{SgprB128}, {IntrId, SgprB128}})
|
|
.Any({{UniB256}, {{SgprB256}, {IntrId, SgprB256}}})
|
|
.Any({{DivB256}, {{VgprB256}, {IntrId, VgprB256}}})
|
|
.Any({{UniB512}, {{SgprB512}, {IntrId, SgprB512}}})
|
|
.Any({{DivB512}, {{VgprB512}, {IntrId, VgprB512}}});
|
|
|
|
} // end initialize rules
|