3428 lines
112 KiB
C++
3428 lines
112 KiB
C++
//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "AMDGPUBaseInfo.h"
|
|
#include "AMDGPU.h"
|
|
#include "AMDGPUAsmUtils.h"
|
|
#include "AMDKernelCodeT.h"
|
|
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
|
|
#include "Utils/AMDKernelCodeTUtils.h"
|
|
#include "llvm/ADT/StringExtras.h"
|
|
#include "llvm/BinaryFormat/ELF.h"
|
|
#include "llvm/IR/Attributes.h"
|
|
#include "llvm/IR/Constants.h"
|
|
#include "llvm/IR/Function.h"
|
|
#include "llvm/IR/GlobalValue.h"
|
|
#include "llvm/IR/IntrinsicsAMDGPU.h"
|
|
#include "llvm/IR/IntrinsicsR600.h"
|
|
#include "llvm/IR/LLVMContext.h"
|
|
#include "llvm/IR/Metadata.h"
|
|
#include "llvm/MC/MCInstrInfo.h"
|
|
#include "llvm/MC/MCRegisterInfo.h"
|
|
#include "llvm/MC/MCSubtargetInfo.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
#include "llvm/TargetParser/TargetParser.h"
|
|
#include <optional>
|
|
|
|
#define GET_INSTRINFO_NAMED_OPS
|
|
#define GET_INSTRMAP_INFO
|
|
#include "AMDGPUGenInstrInfo.inc"
|
|
|
|
static llvm::cl::opt<unsigned> DefaultAMDHSACodeObjectVersion(
|
|
"amdhsa-code-object-version", llvm::cl::Hidden,
|
|
llvm::cl::init(llvm::AMDGPU::AMDHSA_COV6),
|
|
llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "
|
|
"or asm directive still take priority if present)"));
|
|
|
|
namespace {
|
|
|
|
/// \returns Bit mask for given bit \p Shift and bit \p Width.
|
|
unsigned getBitMask(unsigned Shift, unsigned Width) {
|
|
return ((1 << Width) - 1) << Shift;
|
|
}
|
|
|
|
/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
|
|
///
|
|
/// \returns Packed \p Dst.
|
|
unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
|
|
unsigned Mask = getBitMask(Shift, Width);
|
|
return ((Src << Shift) & Mask) | (Dst & ~Mask);
|
|
}
|
|
|
|
/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
|
|
///
|
|
/// \returns Unpacked bits.
|
|
unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
|
|
return (Src & getBitMask(Shift, Width)) >> Shift;
|
|
}
|
|
|
|
/// \returns Vmcnt bit shift (lower bits).
|
|
unsigned getVmcntBitShiftLo(unsigned VersionMajor) {
|
|
return VersionMajor >= 11 ? 10 : 0;
|
|
}
|
|
|
|
/// \returns Vmcnt bit width (lower bits).
|
|
unsigned getVmcntBitWidthLo(unsigned VersionMajor) {
|
|
return VersionMajor >= 11 ? 6 : 4;
|
|
}
|
|
|
|
/// \returns Expcnt bit shift.
|
|
unsigned getExpcntBitShift(unsigned VersionMajor) {
|
|
return VersionMajor >= 11 ? 0 : 4;
|
|
}
|
|
|
|
/// \returns Expcnt bit width.
|
|
unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; }
|
|
|
|
/// \returns Lgkmcnt bit shift.
|
|
unsigned getLgkmcntBitShift(unsigned VersionMajor) {
|
|
return VersionMajor >= 11 ? 4 : 8;
|
|
}
|
|
|
|
/// \returns Lgkmcnt bit width.
|
|
unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
|
|
return VersionMajor >= 10 ? 6 : 4;
|
|
}
|
|
|
|
/// \returns Vmcnt bit shift (higher bits).
|
|
unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; }
|
|
|
|
/// \returns Vmcnt bit width (higher bits).
|
|
unsigned getVmcntBitWidthHi(unsigned VersionMajor) {
|
|
return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0;
|
|
}
|
|
|
|
/// \returns Loadcnt bit width
|
|
unsigned getLoadcntBitWidth(unsigned VersionMajor) {
|
|
return VersionMajor >= 12 ? 6 : 0;
|
|
}
|
|
|
|
/// \returns Samplecnt bit width.
|
|
unsigned getSamplecntBitWidth(unsigned VersionMajor) {
|
|
return VersionMajor >= 12 ? 6 : 0;
|
|
}
|
|
|
|
/// \returns Bvhcnt bit width.
|
|
unsigned getBvhcntBitWidth(unsigned VersionMajor) {
|
|
return VersionMajor >= 12 ? 3 : 0;
|
|
}
|
|
|
|
/// \returns Dscnt bit width.
|
|
unsigned getDscntBitWidth(unsigned VersionMajor) {
|
|
return VersionMajor >= 12 ? 6 : 0;
|
|
}
|
|
|
|
/// \returns Dscnt bit shift in combined S_WAIT instructions.
|
|
unsigned getDscntBitShift(unsigned VersionMajor) { return 0; }
|
|
|
|
/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.
|
|
unsigned getStorecntBitWidth(unsigned VersionMajor) {
|
|
return VersionMajor >= 10 ? 6 : 0;
|
|
}
|
|
|
|
/// \returns Kmcnt bit width.
|
|
unsigned getKmcntBitWidth(unsigned VersionMajor) {
|
|
return VersionMajor >= 12 ? 5 : 0;
|
|
}
|
|
|
|
/// \returns Xcnt bit width.
|
|
unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
|
|
return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
|
|
}
|
|
|
|
/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
|
|
unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
|
|
return VersionMajor >= 12 ? 8 : 0;
|
|
}
|
|
|
|
/// \returns VaSdst bit width
|
|
inline unsigned getVaSdstBitWidth() { return 3; }
|
|
|
|
/// \returns VaSdst bit shift
|
|
inline unsigned getVaSdstBitShift() { return 9; }
|
|
|
|
/// \returns VmVsrc bit width
|
|
inline unsigned getVmVsrcBitWidth() { return 3; }
|
|
|
|
/// \returns VmVsrc bit shift
|
|
inline unsigned getVmVsrcBitShift() { return 2; }
|
|
|
|
/// \returns VaVdst bit width
|
|
inline unsigned getVaVdstBitWidth() { return 4; }
|
|
|
|
/// \returns VaVdst bit shift
|
|
inline unsigned getVaVdstBitShift() { return 12; }
|
|
|
|
/// \returns VaVcc bit width
|
|
inline unsigned getVaVccBitWidth() { return 1; }
|
|
|
|
/// \returns VaVcc bit shift
|
|
inline unsigned getVaVccBitShift() { return 1; }
|
|
|
|
/// \returns SaSdst bit width
|
|
inline unsigned getSaSdstBitWidth() { return 1; }
|
|
|
|
/// \returns SaSdst bit shift
|
|
inline unsigned getSaSdstBitShift() { return 0; }
|
|
|
|
/// \returns VaSsrc width
|
|
inline unsigned getVaSsrcBitWidth() { return 1; }
|
|
|
|
/// \returns VaSsrc bit shift
|
|
inline unsigned getVaSsrcBitShift() { return 8; }
|
|
|
|
/// \returns HoldCnt bit shift
|
|
inline unsigned getHoldCntWidth() { return 1; }
|
|
|
|
/// \returns HoldCnt bit shift
|
|
inline unsigned getHoldCntBitShift() { return 7; }
|
|
|
|
} // end anonymous namespace
|
|
|
|
namespace llvm {
|
|
|
|
namespace AMDGPU {
|
|
|
|
/// \returns true if the target supports signed immediate offset for SMRD
|
|
/// instructions.
|
|
bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
|
|
return isGFX9Plus(ST);
|
|
}
|
|
|
|
/// \returns True if \p STI is AMDHSA.
|
|
bool isHsaAbi(const MCSubtargetInfo &STI) {
|
|
return STI.getTargetTriple().getOS() == Triple::AMDHSA;
|
|
}
|
|
|
|
unsigned getAMDHSACodeObjectVersion(const Module &M) {
|
|
if (auto *Ver = mdconst::extract_or_null<ConstantInt>(
|
|
M.getModuleFlag("amdhsa_code_object_version"))) {
|
|
return (unsigned)Ver->getZExtValue() / 100;
|
|
}
|
|
|
|
return getDefaultAMDHSACodeObjectVersion();
|
|
}
|
|
|
|
unsigned getDefaultAMDHSACodeObjectVersion() {
|
|
return DefaultAMDHSACodeObjectVersion;
|
|
}
|
|
|
|
unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {
|
|
switch (ABIVersion) {
|
|
case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
|
|
return 4;
|
|
case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
|
|
return 5;
|
|
case ELF::ELFABIVERSION_AMDGPU_HSA_V6:
|
|
return 6;
|
|
default:
|
|
return getDefaultAMDHSACodeObjectVersion();
|
|
}
|
|
}
|
|
|
|
uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {
|
|
if (T.getOS() != Triple::AMDHSA)
|
|
return 0;
|
|
|
|
switch (CodeObjectVersion) {
|
|
case 4:
|
|
return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
|
|
case 5:
|
|
return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
|
|
case 6:
|
|
return ELF::ELFABIVERSION_AMDGPU_HSA_V6;
|
|
default:
|
|
report_fatal_error("Unsupported AMDHSA Code Object Version " +
|
|
Twine(CodeObjectVersion));
|
|
}
|
|
}
|
|
|
|
unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {
|
|
switch (CodeObjectVersion) {
|
|
case AMDHSA_COV4:
|
|
return 48;
|
|
case AMDHSA_COV5:
|
|
case AMDHSA_COV6:
|
|
default:
|
|
return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET;
|
|
}
|
|
}
|
|
|
|
// FIXME: All such magic numbers about the ABI should be in a
|
|
// central TD file.
|
|
unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {
|
|
switch (CodeObjectVersion) {
|
|
case AMDHSA_COV4:
|
|
return 24;
|
|
case AMDHSA_COV5:
|
|
case AMDHSA_COV6:
|
|
default:
|
|
return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
|
|
}
|
|
}
|
|
|
|
unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {
|
|
switch (CodeObjectVersion) {
|
|
case AMDHSA_COV4:
|
|
return 32;
|
|
case AMDHSA_COV5:
|
|
case AMDHSA_COV6:
|
|
default:
|
|
return AMDGPU::ImplicitArg::DEFAULT_QUEUE_OFFSET;
|
|
}
|
|
}
|
|
|
|
unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {
|
|
switch (CodeObjectVersion) {
|
|
case AMDHSA_COV4:
|
|
return 40;
|
|
case AMDHSA_COV5:
|
|
case AMDHSA_COV6:
|
|
default:
|
|
return AMDGPU::ImplicitArg::COMPLETION_ACTION_OFFSET;
|
|
}
|
|
}
|
|
|
|
#define GET_MIMGBaseOpcodesTable_IMPL
|
|
#define GET_MIMGDimInfoTable_IMPL
|
|
#define GET_MIMGInfoTable_IMPL
|
|
#define GET_MIMGLZMappingTable_IMPL
|
|
#define GET_MIMGMIPMappingTable_IMPL
|
|
#define GET_MIMGBiasMappingTable_IMPL
|
|
#define GET_MIMGOffsetMappingTable_IMPL
|
|
#define GET_MIMGG16MappingTable_IMPL
|
|
#define GET_MAIInstInfoTable_IMPL
|
|
#define GET_WMMAInstInfoTable_IMPL
|
|
#include "AMDGPUGenSearchableTables.inc"
|
|
|
|
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
|
|
unsigned VDataDwords, unsigned VAddrDwords) {
|
|
const MIMGInfo *Info =
|
|
getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords);
|
|
return Info ? Info->Opcode : -1;
|
|
}
|
|
|
|
const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
|
|
const MIMGInfo *Info = getMIMGInfo(Opc);
|
|
return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
|
|
}
|
|
|
|
int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
|
|
const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
|
|
const MIMGInfo *NewInfo =
|
|
getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,
|
|
NewChannels, OrigInfo->VAddrDwords);
|
|
return NewInfo ? NewInfo->Opcode : -1;
|
|
}
|
|
|
|
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
|
|
const MIMGDimInfo *Dim, bool IsA16,
|
|
bool IsG16Supported) {
|
|
unsigned AddrWords = BaseOpcode->NumExtraArgs;
|
|
unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
|
|
(BaseOpcode->LodOrClampOrMip ? 1 : 0);
|
|
if (IsA16)
|
|
AddrWords += divideCeil(AddrComponents, 2);
|
|
else
|
|
AddrWords += AddrComponents;
|
|
|
|
// Note: For subtargets that support A16 but not G16, enabling A16 also
|
|
// enables 16 bit gradients.
|
|
// For subtargets that support A16 (operand) and G16 (done with a different
|
|
// instruction encoding), they are independent.
|
|
|
|
if (BaseOpcode->Gradients) {
|
|
if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
|
|
// There are two gradients per coordinate, we pack them separately.
|
|
// For the 3d case,
|
|
// we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
|
|
AddrWords += alignTo<2>(Dim->NumGradients / 2);
|
|
else
|
|
AddrWords += Dim->NumGradients;
|
|
}
|
|
return AddrWords;
|
|
}
|
|
|
|
struct MUBUFInfo {
|
|
uint16_t Opcode;
|
|
uint16_t BaseOpcode;
|
|
uint8_t elements;
|
|
bool has_vaddr;
|
|
bool has_srsrc;
|
|
bool has_soffset;
|
|
bool IsBufferInv;
|
|
bool tfe;
|
|
};
|
|
|
|
struct MTBUFInfo {
|
|
uint16_t Opcode;
|
|
uint16_t BaseOpcode;
|
|
uint8_t elements;
|
|
bool has_vaddr;
|
|
bool has_srsrc;
|
|
bool has_soffset;
|
|
};
|
|
|
|
struct SMInfo {
|
|
uint16_t Opcode;
|
|
bool IsBuffer;
|
|
};
|
|
|
|
struct VOPInfo {
|
|
uint16_t Opcode;
|
|
bool IsSingle;
|
|
};
|
|
|
|
struct VOPC64DPPInfo {
|
|
uint16_t Opcode;
|
|
};
|
|
|
|
struct VOPCDPPAsmOnlyInfo {
|
|
uint16_t Opcode;
|
|
};
|
|
|
|
struct VOP3CDPPAsmOnlyInfo {
|
|
uint16_t Opcode;
|
|
};
|
|
|
|
struct VOPDComponentInfo {
|
|
uint16_t BaseVOP;
|
|
uint16_t VOPDOp;
|
|
bool CanBeVOPDX;
|
|
bool CanBeVOPD3X;
|
|
};
|
|
|
|
struct VOPDInfo {
|
|
uint16_t Opcode;
|
|
uint16_t OpX;
|
|
uint16_t OpY;
|
|
uint16_t Subtarget;
|
|
bool VOPD3;
|
|
};
|
|
|
|
struct VOPTrue16Info {
|
|
uint16_t Opcode;
|
|
bool IsTrue16;
|
|
};
|
|
|
|
#define GET_FP4FP8DstByteSelTable_DECL
|
|
#define GET_FP4FP8DstByteSelTable_IMPL
|
|
|
|
struct DPMACCInstructionInfo {
|
|
uint16_t Opcode;
|
|
bool IsDPMACCInstruction;
|
|
};
|
|
|
|
struct FP4FP8DstByteSelInfo {
|
|
uint16_t Opcode;
|
|
bool HasFP8DstByteSel;
|
|
bool HasFP4DstByteSel;
|
|
};
|
|
|
|
#define GET_MTBUFInfoTable_DECL
|
|
#define GET_MTBUFInfoTable_IMPL
|
|
#define GET_MUBUFInfoTable_DECL
|
|
#define GET_MUBUFInfoTable_IMPL
|
|
#define GET_SMInfoTable_DECL
|
|
#define GET_SMInfoTable_IMPL
|
|
#define GET_VOP1InfoTable_DECL
|
|
#define GET_VOP1InfoTable_IMPL
|
|
#define GET_VOP2InfoTable_DECL
|
|
#define GET_VOP2InfoTable_IMPL
|
|
#define GET_VOP3InfoTable_DECL
|
|
#define GET_VOP3InfoTable_IMPL
|
|
#define GET_VOPC64DPPTable_DECL
|
|
#define GET_VOPC64DPPTable_IMPL
|
|
#define GET_VOPC64DPP8Table_DECL
|
|
#define GET_VOPC64DPP8Table_IMPL
|
|
#define GET_VOPCAsmOnlyInfoTable_DECL
|
|
#define GET_VOPCAsmOnlyInfoTable_IMPL
|
|
#define GET_VOP3CAsmOnlyInfoTable_DECL
|
|
#define GET_VOP3CAsmOnlyInfoTable_IMPL
|
|
#define GET_VOPDComponentTable_DECL
|
|
#define GET_VOPDComponentTable_IMPL
|
|
#define GET_VOPDPairs_DECL
|
|
#define GET_VOPDPairs_IMPL
|
|
#define GET_VOPTrue16Table_DECL
|
|
#define GET_VOPTrue16Table_IMPL
|
|
#define GET_True16D16Table_IMPL
|
|
#define GET_WMMAOpcode2AddrMappingTable_DECL
|
|
#define GET_WMMAOpcode2AddrMappingTable_IMPL
|
|
#define GET_WMMAOpcode3AddrMappingTable_DECL
|
|
#define GET_WMMAOpcode3AddrMappingTable_IMPL
|
|
#define GET_getMFMA_F8F6F4_WithSize_DECL
|
|
#define GET_getMFMA_F8F6F4_WithSize_IMPL
|
|
#define GET_isMFMA_F8F6F4Table_IMPL
|
|
#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
|
|
|
|
#include "AMDGPUGenSearchableTables.inc"
|
|
|
|
int getMTBUFBaseOpcode(unsigned Opc) {
|
|
const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
|
|
return Info ? Info->BaseOpcode : -1;
|
|
}
|
|
|
|
int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
|
|
const MTBUFInfo *Info =
|
|
getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
|
|
return Info ? Info->Opcode : -1;
|
|
}
|
|
|
|
int getMTBUFElements(unsigned Opc) {
|
|
const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
|
|
return Info ? Info->elements : 0;
|
|
}
|
|
|
|
bool getMTBUFHasVAddr(unsigned Opc) {
|
|
const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
|
|
return Info && Info->has_vaddr;
|
|
}
|
|
|
|
bool getMTBUFHasSrsrc(unsigned Opc) {
|
|
const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
|
|
return Info && Info->has_srsrc;
|
|
}
|
|
|
|
bool getMTBUFHasSoffset(unsigned Opc) {
|
|
const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
|
|
return Info && Info->has_soffset;
|
|
}
|
|
|
|
int getMUBUFBaseOpcode(unsigned Opc) {
|
|
const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
|
|
return Info ? Info->BaseOpcode : -1;
|
|
}
|
|
|
|
int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
|
|
const MUBUFInfo *Info =
|
|
getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
|
|
return Info ? Info->Opcode : -1;
|
|
}
|
|
|
|
int getMUBUFElements(unsigned Opc) {
|
|
const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
|
|
return Info ? Info->elements : 0;
|
|
}
|
|
|
|
bool getMUBUFHasVAddr(unsigned Opc) {
|
|
const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
|
|
return Info && Info->has_vaddr;
|
|
}
|
|
|
|
bool getMUBUFHasSrsrc(unsigned Opc) {
|
|
const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
|
|
return Info && Info->has_srsrc;
|
|
}
|
|
|
|
bool getMUBUFHasSoffset(unsigned Opc) {
|
|
const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
|
|
return Info && Info->has_soffset;
|
|
}
|
|
|
|
bool getMUBUFIsBufferInv(unsigned Opc) {
|
|
const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
|
|
return Info && Info->IsBufferInv;
|
|
}
|
|
|
|
bool getMUBUFTfe(unsigned Opc) {
|
|
const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
|
|
return Info && Info->tfe;
|
|
}
|
|
|
|
bool getSMEMIsBuffer(unsigned Opc) {
|
|
const SMInfo *Info = getSMEMOpcodeHelper(Opc);
|
|
return Info && Info->IsBuffer;
|
|
}
|
|
|
|
bool getVOP1IsSingle(unsigned Opc) {
|
|
const VOPInfo *Info = getVOP1OpcodeHelper(Opc);
|
|
return !Info || Info->IsSingle;
|
|
}
|
|
|
|
bool getVOP2IsSingle(unsigned Opc) {
|
|
const VOPInfo *Info = getVOP2OpcodeHelper(Opc);
|
|
return !Info || Info->IsSingle;
|
|
}
|
|
|
|
bool getVOP3IsSingle(unsigned Opc) {
|
|
const VOPInfo *Info = getVOP3OpcodeHelper(Opc);
|
|
return !Info || Info->IsSingle;
|
|
}
|
|
|
|
bool isVOPC64DPP(unsigned Opc) {
|
|
return isVOPC64DPPOpcodeHelper(Opc) || isVOPC64DPP8OpcodeHelper(Opc);
|
|
}
|
|
|
|
bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opc); }
|
|
|
|
bool getMAIIsDGEMM(unsigned Opc) {
|
|
const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
|
|
return Info && Info->is_dgemm;
|
|
}
|
|
|
|
bool getMAIIsGFX940XDL(unsigned Opc) {
|
|
const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
|
|
return Info && Info->is_gfx940_xdl;
|
|
}
|
|
|
|
bool getWMMAIsXDL(unsigned Opc) {
|
|
const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opc);
|
|
return Info ? Info->is_wmma_xdl : false;
|
|
}
|
|
|
|
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
|
|
switch (EncodingVal) {
|
|
case MFMAScaleFormats::FP6_E2M3:
|
|
case MFMAScaleFormats::FP6_E3M2:
|
|
return 6;
|
|
case MFMAScaleFormats::FP4_E2M1:
|
|
return 4;
|
|
case MFMAScaleFormats::FP8_E4M3:
|
|
case MFMAScaleFormats::FP8_E5M2:
|
|
default:
|
|
return 8;
|
|
}
|
|
|
|
llvm_unreachable("covered switch over mfma scale formats");
|
|
}
|
|
|
|
const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,
|
|
unsigned BLGP,
|
|
unsigned F8F8Opcode) {
|
|
uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(CBSZ);
|
|
uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(BLGP);
|
|
return getMFMA_F8F6F4_InstWithNumRegs(SrcANumRegs, SrcBNumRegs, F8F8Opcode);
|
|
}
|
|
|
|
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt) {
|
|
switch (Fmt) {
|
|
case WMMA::MATRIX_FMT_FP8:
|
|
case WMMA::MATRIX_FMT_BF8:
|
|
return 16;
|
|
case WMMA::MATRIX_FMT_FP6:
|
|
case WMMA::MATRIX_FMT_BF6:
|
|
return 12;
|
|
case WMMA::MATRIX_FMT_FP4:
|
|
return 8;
|
|
}
|
|
|
|
llvm_unreachable("covered switch over wmma scale formats");
|
|
}
|
|
|
|
const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,
|
|
unsigned FmtB,
|
|
unsigned F8F8Opcode) {
|
|
uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(FmtA);
|
|
uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(FmtB);
|
|
return getMFMA_F8F6F4_InstWithNumRegs(SrcANumRegs, SrcBNumRegs, F8F8Opcode);
|
|
}
|
|
|
|
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST) {
|
|
if (ST.hasFeature(AMDGPU::FeatureGFX1250Insts))
|
|
return SIEncodingFamily::GFX1250;
|
|
if (ST.hasFeature(AMDGPU::FeatureGFX12Insts))
|
|
return SIEncodingFamily::GFX12;
|
|
if (ST.hasFeature(AMDGPU::FeatureGFX11Insts))
|
|
return SIEncodingFamily::GFX11;
|
|
llvm_unreachable("Subtarget generation does not support VOPD!");
|
|
}
|
|
|
|
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {
|
|
bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
|
|
Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
|
|
const VOPDComponentInfo *Info = getVOPDComponentHelper(Opc);
|
|
if (Info) {
|
|
// Check that Opc can be used as VOPDY for this encoding. V_MOV_B32 as a
|
|
// VOPDX is just a placeholder here, it is supported on all encodings.
|
|
// TODO: This can be optimized by creating tables of supported VOPDY
|
|
// opcodes per encoding.
|
|
unsigned VOPDMov = AMDGPU::getVOPDOpcode(AMDGPU::V_MOV_B32_e32, VOPD3);
|
|
bool CanBeVOPDY = getVOPDFull(VOPDMov, AMDGPU::getVOPDOpcode(Opc, VOPD3),
|
|
EncodingFamily, VOPD3) != -1;
|
|
return {VOPD3 ? Info->CanBeVOPD3X : Info->CanBeVOPDX, CanBeVOPDY};
|
|
}
|
|
|
|
return {false, false};
|
|
}
|
|
|
|
unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {
|
|
bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
|
|
Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
|
|
const VOPDComponentInfo *Info = getVOPDComponentHelper(Opc);
|
|
return Info ? Info->VOPDOp : ~0u;
|
|
}
|
|
|
|
bool isVOPD(unsigned Opc) {
|
|
return AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::src0X);
|
|
}
|
|
|
|
bool isMAC(unsigned Opc) {
|
|
return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
|
|
Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
|
|
Opc == AMDGPU::V_MAC_F32_e64_vi ||
|
|
Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
|
|
Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
|
|
Opc == AMDGPU::V_MAC_F16_e64_vi ||
|
|
Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
|
|
Opc == AMDGPU::V_FMAC_F64_e64_gfx12 ||
|
|
Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
|
|
Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||
|
|
Opc == AMDGPU::V_FMAC_F32_e64_gfx12 ||
|
|
Opc == AMDGPU::V_FMAC_F32_e64_vi ||
|
|
Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
|
|
Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||
|
|
Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||
|
|
Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 ||
|
|
Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 ||
|
|
Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 ||
|
|
Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 ||
|
|
Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi ||
|
|
Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi ||
|
|
Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi ||
|
|
Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi ||
|
|
Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;
|
|
}
|
|
|
|
bool isPermlane16(unsigned Opc) {
|
|
return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
|
|
Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 ||
|
|
Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 ||
|
|
Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 ||
|
|
Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 ||
|
|
Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 ||
|
|
Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 ||
|
|
Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12;
|
|
}
|
|
|
|
bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc) {
|
|
return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 ||
|
|
Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;
|
|
}
|
|
|
|
bool isGenericAtomic(unsigned Opc) {
|
|
return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX ||
|
|
Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP ||
|
|
Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
|
|
}
|
|
|
|
bool isAsyncStore(unsigned Opc) {
|
|
return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 ||
|
|
Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 ||
|
|
Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 ||
|
|
Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 ||
|
|
Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 ||
|
|
Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 ||
|
|
Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 ||
|
|
Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;
|
|
}
|
|
|
|
bool isTensorStore(unsigned Opc) {
|
|
return Opc == TENSOR_STORE_FROM_LDS_gfx1250 ||
|
|
Opc == TENSOR_STORE_FROM_LDS_D2_gfx1250;
|
|
}
|
|
|
|
unsigned getTemporalHintType(const MCInstrDesc TID) {
|
|
if (TID.TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet))
|
|
return CPol::TH_TYPE_ATOMIC;
|
|
unsigned Opc = TID.getOpcode();
|
|
// Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
|
|
if (TID.mayStore() &&
|
|
(isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
|
|
return CPol::TH_TYPE_STORE;
|
|
|
|
// This will default to returning TH_TYPE_LOAD when neither MayStore nor
|
|
// MayLoad flag is present which is the case with instructions like
|
|
// image_get_resinfo.
|
|
return CPol::TH_TYPE_LOAD;
|
|
}
|
|
|
|
bool isTrue16Inst(unsigned Opc) {
|
|
const VOPTrue16Info *Info = getTrue16OpcodeHelper(Opc);
|
|
return Info && Info->IsTrue16;
|
|
}
|
|
|
|
FPType getFPDstSelType(unsigned Opc) {
|
|
const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opc);
|
|
if (!Info)
|
|
return FPType::None;
|
|
if (Info->HasFP8DstByteSel)
|
|
return FPType::FP8;
|
|
if (Info->HasFP4DstByteSel)
|
|
return FPType::FP4;
|
|
|
|
return FPType::None;
|
|
}
|
|
|
|
unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {
|
|
const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opc);
|
|
return Info ? Info->Opcode3Addr : ~0u;
|
|
}
|
|
|
|
unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc) {
|
|
const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom3AddrOpcode(Opc);
|
|
return Info ? Info->Opcode2Addr : ~0u;
|
|
}
|
|
|
|
// Wrapper for Tablegen'd function. enum Subtarget is not defined in any
|
|
// header files, so we need to wrap it in a function that takes unsigned
|
|
// instead.
|
|
int getMCOpcode(uint16_t Opcode, unsigned Gen) {
|
|
return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
|
|
}
|
|
|
|
unsigned getBitOp2(unsigned Opc) {
|
|
switch (Opc) {
|
|
default:
|
|
return 0;
|
|
case AMDGPU::V_AND_B32_e32:
|
|
return 0x40;
|
|
case AMDGPU::V_OR_B32_e32:
|
|
return 0x54;
|
|
case AMDGPU::V_XOR_B32_e32:
|
|
return 0x14;
|
|
case AMDGPU::V_XNOR_B32_e32:
|
|
return 0x41;
|
|
}
|
|
}
|
|
|
|
int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
|
|
bool VOPD3) {
|
|
bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(OpY) : 0;
|
|
OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;
|
|
const VOPDInfo *Info =
|
|
getVOPDInfoFromComponentOpcodes(OpX, OpY, EncodingFamily, VOPD3);
|
|
return Info ? Info->Opcode : -1;
|
|
}
|
|
|
|
std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
|
|
const VOPDInfo *Info = getVOPDOpcodeHelper(VOPDOpcode);
|
|
assert(Info);
|
|
const auto *OpX = getVOPDBaseFromComponent(Info->OpX);
|
|
const auto *OpY = getVOPDBaseFromComponent(Info->OpY);
|
|
assert(OpX && OpY);
|
|
return {OpX->BaseVOP, OpY->BaseVOP};
|
|
}
|
|
|
|
namespace VOPD {
|
|
|
|
ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {
|
|
assert(OpDesc.getNumDefs() == Component::DST_NUM);
|
|
|
|
assert(OpDesc.getOperandConstraint(Component::SRC0, MCOI::TIED_TO) == -1);
|
|
assert(OpDesc.getOperandConstraint(Component::SRC1, MCOI::TIED_TO) == -1);
|
|
auto TiedIdx = OpDesc.getOperandConstraint(Component::SRC2, MCOI::TIED_TO);
|
|
assert(TiedIdx == -1 || TiedIdx == Component::DST);
|
|
HasSrc2Acc = TiedIdx != -1;
|
|
Opcode = OpDesc.getOpcode();
|
|
|
|
IsVOP3 = VOP3Layout || (OpDesc.TSFlags & SIInstrFlags::VOP3);
|
|
SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src2) ? 3
|
|
: AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::imm) ? 3
|
|
: AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1) ? 2
|
|
: 1;
|
|
assert(SrcOperandsNum <= Component::MAX_SRC_NUM);
|
|
|
|
if (Opcode == AMDGPU::V_CNDMASK_B32_e32 ||
|
|
Opcode == AMDGPU::V_CNDMASK_B32_e64) {
|
|
// CNDMASK is an awkward exception, it has FP modifiers, but not FP
|
|
// operands.
|
|
NumVOPD3Mods = 2;
|
|
if (IsVOP3)
|
|
SrcOperandsNum = 3;
|
|
} else if (isSISrcFPOperand(OpDesc,
|
|
getNamedOperandIdx(Opcode, OpName::src0))) {
|
|
// All FP VOPD instructions have Neg modifiers for all operands except
|
|
// for tied src2.
|
|
NumVOPD3Mods = SrcOperandsNum;
|
|
if (HasSrc2Acc)
|
|
--NumVOPD3Mods;
|
|
}
|
|
|
|
if (OpDesc.TSFlags & SIInstrFlags::VOP3)
|
|
return;
|
|
|
|
auto OperandsNum = OpDesc.getNumOperands();
|
|
unsigned CompOprIdx;
|
|
for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {
|
|
if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {
|
|
MandatoryLiteralIdx = CompOprIdx;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
int ComponentProps::getBitOp3OperandIdx() const {
|
|
return getNamedOperandIdx(Opcode, OpName::bitop3);
|
|
}
|
|
|
|
unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {
|
|
assert(CompOprIdx < Component::MAX_OPR_NUM);
|
|
|
|
if (CompOprIdx == Component::DST)
|
|
return getIndexOfDstInParsedOperands();
|
|
|
|
auto CompSrcIdx = CompOprIdx - Component::DST_NUM;
|
|
if (CompSrcIdx < getCompParsedSrcOperandsNum())
|
|
return getIndexOfSrcInParsedOperands(CompSrcIdx);
|
|
|
|
// The specified operand does not exist.
|
|
return 0;
|
|
}
|
|
|
|
std::optional<unsigned> InstInfo::getInvalidCompOperandIndex(
|
|
std::function<unsigned(unsigned, unsigned)> GetRegIdx,
|
|
const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR,
|
|
bool VOPD3) const {
|
|
|
|
auto OpXRegs = getRegIndices(ComponentIndex::X, GetRegIdx,
|
|
CompInfo[ComponentIndex::X].isVOP3());
|
|
auto OpYRegs = getRegIndices(ComponentIndex::Y, GetRegIdx,
|
|
CompInfo[ComponentIndex::Y].isVOP3());
|
|
|
|
const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,
|
|
unsigned BanksMask) -> bool {
|
|
MCRegister BaseX = MRI.getSubReg(X, AMDGPU::sub0);
|
|
MCRegister BaseY = MRI.getSubReg(Y, AMDGPU::sub0);
|
|
if (!BaseX)
|
|
BaseX = X;
|
|
if (!BaseY)
|
|
BaseY = Y;
|
|
if ((BaseX & BanksMask) == (BaseY & BanksMask))
|
|
return true;
|
|
if (BaseX != X /* This is 64-bit register */ &&
|
|
((BaseX + 1) & BanksMask) == (BaseY & BanksMask))
|
|
return true;
|
|
if (BaseY != Y && (BaseX & BanksMask) == ((BaseY + 1) & BanksMask))
|
|
return true;
|
|
|
|
// If both are 64-bit bank conflict will be detected yet while checking
|
|
// the first subreg.
|
|
return false;
|
|
};
|
|
|
|
unsigned CompOprIdx;
|
|
for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {
|
|
unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]
|
|
: VOPD_VGPR_BANK_MASKS[CompOprIdx];
|
|
if (!OpXRegs[CompOprIdx] || !OpYRegs[CompOprIdx])
|
|
continue;
|
|
|
|
if (SkipSrc && CompOprIdx >= Component::DST_NUM)
|
|
continue;
|
|
|
|
if (CompOprIdx < Component::DST_NUM) {
|
|
// Even if we do not check vdst parity, vdst operands still shall not
|
|
// overlap.
|
|
if (MRI.regsOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx]))
|
|
return CompOprIdx;
|
|
if (VOPD3) // No need to check dst parity.
|
|
continue;
|
|
}
|
|
|
|
if (banksOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx], BanksMasks) &&
|
|
(!AllowSameVGPR || CompOprIdx < Component::DST_NUM ||
|
|
OpXRegs[CompOprIdx] != OpYRegs[CompOprIdx]))
|
|
return CompOprIdx;
|
|
}
|
|
|
|
return {};
|
|
}
|
|
|
|
// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used
|
|
// by the specified component. If an operand is unused
|
|
// or is not a VGPR, the corresponding value is 0.
|
|
//
|
|
// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
|
|
// for the specified component and MC operand. The callback must return 0
|
|
// if the operand is not a register or not a VGPR.
|
|
InstInfo::RegIndices
|
|
InstInfo::getRegIndices(unsigned CompIdx,
|
|
std::function<unsigned(unsigned, unsigned)> GetRegIdx,
|
|
bool VOPD3) const {
|
|
assert(CompIdx < COMPONENTS_NUM);
|
|
|
|
const auto &Comp = CompInfo[CompIdx];
|
|
InstInfo::RegIndices RegIndices;
|
|
|
|
RegIndices[DST] = GetRegIdx(CompIdx, Comp.getIndexOfDstInMCOperands());
|
|
|
|
for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {
|
|
unsigned CompSrcIdx = CompOprIdx - DST_NUM;
|
|
RegIndices[CompOprIdx] =
|
|
Comp.hasRegSrcOperand(CompSrcIdx)
|
|
? GetRegIdx(CompIdx,
|
|
Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))
|
|
: 0;
|
|
}
|
|
return RegIndices;
|
|
}
|
|
|
|
} // namespace VOPD
|
|
|
|
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY) {
|
|
return VOPD::InstInfo(OpX, OpY);
|
|
}
|
|
|
|
VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
|
|
const MCInstrInfo *InstrInfo) {
|
|
auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);
|
|
const auto &OpXDesc = InstrInfo->get(OpX);
|
|
const auto &OpYDesc = InstrInfo->get(OpY);
|
|
bool VOPD3 = InstrInfo->get(VOPDOpcode).TSFlags & SIInstrFlags::VOPD3;
|
|
VOPD::ComponentInfo OpXInfo(OpXDesc, VOPD::ComponentKind::COMPONENT_X, VOPD3);
|
|
VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);
|
|
return VOPD::InstInfo(OpXInfo, OpYInfo);
|
|
}
|
|
|
|
namespace IsaInfo {
|
|
|
|
AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI)
|
|
: STI(STI), XnackSetting(TargetIDSetting::Any),
|
|
SramEccSetting(TargetIDSetting::Any) {
|
|
if (!STI.getFeatureBits().test(FeatureSupportsXNACK))
|
|
XnackSetting = TargetIDSetting::Unsupported;
|
|
if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC))
|
|
SramEccSetting = TargetIDSetting::Unsupported;
|
|
}
|
|
|
|
void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) {
|
|
// Check if xnack or sramecc is explicitly enabled or disabled. In the
|
|
// absence of the target features we assume we must generate code that can run
|
|
// in any environment.
|
|
SubtargetFeatures Features(FS);
|
|
std::optional<bool> XnackRequested;
|
|
std::optional<bool> SramEccRequested;
|
|
|
|
for (const std::string &Feature : Features.getFeatures()) {
|
|
if (Feature == "+xnack")
|
|
XnackRequested = true;
|
|
else if (Feature == "-xnack")
|
|
XnackRequested = false;
|
|
else if (Feature == "+sramecc")
|
|
SramEccRequested = true;
|
|
else if (Feature == "-sramecc")
|
|
SramEccRequested = false;
|
|
}
|
|
|
|
bool XnackSupported = isXnackSupported();
|
|
bool SramEccSupported = isSramEccSupported();
|
|
|
|
if (XnackRequested) {
|
|
if (XnackSupported) {
|
|
XnackSetting =
|
|
*XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off;
|
|
} else {
|
|
// If a specific xnack setting was requested and this GPU does not support
|
|
// xnack emit a warning. Setting will remain set to "Unsupported".
|
|
if (*XnackRequested) {
|
|
errs() << "warning: xnack 'On' was requested for a processor that does "
|
|
"not support it!\n";
|
|
} else {
|
|
errs() << "warning: xnack 'Off' was requested for a processor that "
|
|
"does not support it!\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
if (SramEccRequested) {
|
|
if (SramEccSupported) {
|
|
SramEccSetting =
|
|
*SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off;
|
|
} else {
|
|
// If a specific sramecc setting was requested and this GPU does not
|
|
// support sramecc emit a warning. Setting will remain set to
|
|
// "Unsupported".
|
|
if (*SramEccRequested) {
|
|
errs() << "warning: sramecc 'On' was requested for a processor that "
|
|
"does not support it!\n";
|
|
} else {
|
|
errs() << "warning: sramecc 'Off' was requested for a processor that "
|
|
"does not support it!\n";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static TargetIDSetting
|
|
getTargetIDSettingFromFeatureString(StringRef FeatureString) {
|
|
if (FeatureString.ends_with("-"))
|
|
return TargetIDSetting::Off;
|
|
if (FeatureString.ends_with("+"))
|
|
return TargetIDSetting::On;
|
|
|
|
llvm_unreachable("Malformed feature string");
|
|
}
|
|
|
|
void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) {
|
|
SmallVector<StringRef, 3> TargetIDSplit;
|
|
TargetID.split(TargetIDSplit, ':');
|
|
|
|
for (const auto &FeatureString : TargetIDSplit) {
|
|
if (FeatureString.starts_with("xnack"))
|
|
XnackSetting = getTargetIDSettingFromFeatureString(FeatureString);
|
|
if (FeatureString.starts_with("sramecc"))
|
|
SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString);
|
|
}
|
|
}
|
|
|
|
std::string AMDGPUTargetID::toString() const {
|
|
std::string StringRep;
|
|
raw_string_ostream StreamRep(StringRep);
|
|
|
|
auto TargetTriple = STI.getTargetTriple();
|
|
auto Version = getIsaVersion(STI.getCPU());
|
|
|
|
StreamRep << TargetTriple.getArchName() << '-' << TargetTriple.getVendorName()
|
|
<< '-' << TargetTriple.getOSName() << '-'
|
|
<< TargetTriple.getEnvironmentName() << '-';
|
|
|
|
std::string Processor;
|
|
// TODO: Following else statement is present here because we used various
|
|
// alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803').
|
|
// Remove once all aliases are removed from GCNProcessors.td.
|
|
if (Version.Major >= 9)
|
|
Processor = STI.getCPU().str();
|
|
else
|
|
Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) +
|
|
Twine(Version.Stepping))
|
|
.str();
|
|
|
|
std::string Features;
|
|
if (STI.getTargetTriple().getOS() == Triple::AMDHSA) {
|
|
// sramecc.
|
|
if (getSramEccSetting() == TargetIDSetting::Off)
|
|
Features += ":sramecc-";
|
|
else if (getSramEccSetting() == TargetIDSetting::On)
|
|
Features += ":sramecc+";
|
|
// xnack.
|
|
if (getXnackSetting() == TargetIDSetting::Off)
|
|
Features += ":xnack-";
|
|
else if (getXnackSetting() == TargetIDSetting::On)
|
|
Features += ":xnack+";
|
|
}
|
|
|
|
StreamRep << Processor << Features;
|
|
|
|
return StringRep;
|
|
}
|
|
|
|
unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
|
|
if (STI->getFeatureBits().test(FeatureWavefrontSize16))
|
|
return 16;
|
|
if (STI->getFeatureBits().test(FeatureWavefrontSize32))
|
|
return 32;
|
|
|
|
return 64;
|
|
}
|
|
|
|
unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
|
|
unsigned BytesPerCU = getAddressableLocalMemorySize(STI);
|
|
|
|
// "Per CU" really means "per whatever functional block the waves of a
|
|
// workgroup must share". So the effective local memory size is doubled in
|
|
// WGP mode on gfx10.
|
|
if (isGFX10Plus(*STI) && !STI->getFeatureBits().test(FeatureCuMode))
|
|
BytesPerCU *= 2;
|
|
|
|
return BytesPerCU;
|
|
}
|
|
|
|
unsigned getAddressableLocalMemorySize(const MCSubtargetInfo *STI) {
|
|
if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize32768))
|
|
return 32768;
|
|
if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize65536))
|
|
return 65536;
|
|
if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize163840))
|
|
return 163840;
|
|
if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize327680))
|
|
return 327680;
|
|
return 32768;
|
|
}
|
|
|
|
unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
|
|
// "Per CU" really means "per whatever functional block the waves of a
|
|
// workgroup must share".
|
|
|
|
// GFX12.5 only supports CU mode, which contains four SIMDs.
|
|
if (isGFX1250(*STI)) {
|
|
assert(STI->getFeatureBits().test(FeatureCuMode));
|
|
return 4;
|
|
}
|
|
|
|
// For gfx10 in CU mode the functional block is the CU, which contains
|
|
// two SIMDs.
|
|
if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode))
|
|
return 2;
|
|
|
|
// Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP
|
|
// contains two CUs, so a total of four SIMDs.
|
|
return 4;
|
|
}
|
|
|
|
unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
|
|
unsigned FlatWorkGroupSize) {
|
|
assert(FlatWorkGroupSize != 0);
|
|
if (!STI->getTargetTriple().isAMDGCN())
|
|
return 8;
|
|
unsigned MaxWaves = getMaxWavesPerEU(STI) * getEUsPerCU(STI);
|
|
unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
|
|
if (N == 1) {
|
|
// Single-wave workgroups don't consume barrier resources.
|
|
return MaxWaves;
|
|
}
|
|
|
|
unsigned MaxBarriers = 16;
|
|
if (isGFX10Plus(*STI) && !STI->getFeatureBits().test(FeatureCuMode))
|
|
MaxBarriers = 32;
|
|
|
|
return std::min(MaxWaves / N, MaxBarriers);
|
|
}
|
|
|
|
unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { return 1; }
|
|
|
|
unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {
|
|
// FIXME: Need to take scratch memory into account.
|
|
if (isGFX90A(*STI))
|
|
return 8;
|
|
if (!isGFX10Plus(*STI))
|
|
return 10;
|
|
return hasGFX10_3Insts(*STI) ? 16 : 20;
|
|
}
|
|
|
|
unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI,
|
|
unsigned FlatWorkGroupSize) {
|
|
return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),
|
|
getEUsPerCU(STI));
|
|
}
|
|
|
|
unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { return 1; }
|
|
|
|
unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
|
|
// Some subtargets allow encoding 2048, but this isn't tested or supported.
|
|
return 1024;
|
|
}
|
|
|
|
unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
|
|
unsigned FlatWorkGroupSize) {
|
|
return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));
|
|
}
|
|
|
|
unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
|
|
IsaVersion Version = getIsaVersion(STI->getCPU());
|
|
if (Version.Major >= 10)
|
|
return getAddressableNumSGPRs(STI);
|
|
if (Version.Major >= 8)
|
|
return 16;
|
|
return 8;
|
|
}
|
|
|
|
unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { return 8; }
|
|
|
|
unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {
|
|
IsaVersion Version = getIsaVersion(STI->getCPU());
|
|
if (Version.Major >= 8)
|
|
return 800;
|
|
return 512;
|
|
}
|
|
|
|
unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {
|
|
if (STI->getFeatureBits().test(FeatureSGPRInitBug))
|
|
return FIXED_NUM_SGPRS_FOR_INIT_BUG;
|
|
|
|
IsaVersion Version = getIsaVersion(STI->getCPU());
|
|
if (Version.Major >= 10)
|
|
return 106;
|
|
if (Version.Major >= 8)
|
|
return 102;
|
|
return 104;
|
|
}
|
|
|
|
unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
|
|
assert(WavesPerEU != 0);
|
|
|
|
IsaVersion Version = getIsaVersion(STI->getCPU());
|
|
if (Version.Major >= 10)
|
|
return 0;
|
|
|
|
if (WavesPerEU >= getMaxWavesPerEU(STI))
|
|
return 0;
|
|
|
|
unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
|
|
if (STI->getFeatureBits().test(FeatureTrapHandler))
|
|
MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
|
|
MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
|
|
return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
|
|
}
|
|
|
|
unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
|
|
bool Addressable) {
|
|
assert(WavesPerEU != 0);
|
|
|
|
unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
|
|
IsaVersion Version = getIsaVersion(STI->getCPU());
|
|
if (Version.Major >= 10)
|
|
return Addressable ? AddressableNumSGPRs : 108;
|
|
if (Version.Major >= 8 && !Addressable)
|
|
AddressableNumSGPRs = 112;
|
|
unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;
|
|
if (STI->getFeatureBits().test(FeatureTrapHandler))
|
|
MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
|
|
MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));
|
|
return std::min(MaxNumSGPRs, AddressableNumSGPRs);
|
|
}
|
|
|
|
unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
|
|
bool FlatScrUsed, bool XNACKUsed) {
|
|
unsigned ExtraSGPRs = 0;
|
|
if (VCCUsed)
|
|
ExtraSGPRs = 2;
|
|
|
|
IsaVersion Version = getIsaVersion(STI->getCPU());
|
|
if (Version.Major >= 10)
|
|
return ExtraSGPRs;
|
|
|
|
if (Version.Major < 8) {
|
|
if (FlatScrUsed)
|
|
ExtraSGPRs = 4;
|
|
} else {
|
|
if (XNACKUsed)
|
|
ExtraSGPRs = 4;
|
|
|
|
if (FlatScrUsed ||
|
|
STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch))
|
|
ExtraSGPRs = 6;
|
|
}
|
|
|
|
return ExtraSGPRs;
|
|
}
|
|
|
|
unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
|
|
bool FlatScrUsed) {
|
|
return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
|
|
STI->getFeatureBits().test(AMDGPU::FeatureXNACK));
|
|
}
|
|
|
|
static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
|
|
unsigned Granule) {
|
|
return divideCeil(std::max(1u, NumRegs), Granule);
|
|
}
|
|
|
|
unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {
|
|
// SGPRBlocks is actual number of SGPR blocks minus 1.
|
|
return getGranulatedNumRegisterBlocks(NumSGPRs, getSGPREncodingGranule(STI)) -
|
|
1;
|
|
}
|
|
|
|
unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,
|
|
unsigned DynamicVGPRBlockSize,
|
|
std::optional<bool> EnableWavefrontSize32) {
|
|
if (STI->getFeatureBits().test(FeatureGFX90AInsts))
|
|
return 8;
|
|
|
|
if (DynamicVGPRBlockSize != 0)
|
|
return DynamicVGPRBlockSize;
|
|
|
|
// Temporarily check the subtarget feature, until we fully switch to using
|
|
// attributes.
|
|
if (STI->getFeatureBits().test(FeatureDynamicVGPR))
|
|
return STI->getFeatureBits().test(FeatureDynamicVGPRBlockSize32) ? 32 : 16;
|
|
|
|
bool IsWave32 = EnableWavefrontSize32
|
|
? *EnableWavefrontSize32
|
|
: STI->getFeatureBits().test(FeatureWavefrontSize32);
|
|
|
|
if (STI->getFeatureBits().test(Feature1_5xVGPRs))
|
|
return IsWave32 ? 24 : 12;
|
|
|
|
if (hasGFX10_3Insts(*STI))
|
|
return IsWave32 ? 16 : 8;
|
|
|
|
return IsWave32 ? 8 : 4;
|
|
}
|
|
|
|
unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,
|
|
std::optional<bool> EnableWavefrontSize32) {
|
|
if (STI->getFeatureBits().test(FeatureGFX90AInsts))
|
|
return 8;
|
|
|
|
bool IsWave32 = EnableWavefrontSize32
|
|
? *EnableWavefrontSize32
|
|
: STI->getFeatureBits().test(FeatureWavefrontSize32);
|
|
|
|
return IsWave32 ? 8 : 4;
|
|
}
|
|
|
|
unsigned getArchVGPRAllocGranule() { return 4; }
|
|
|
|
unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
|
|
if (STI->getFeatureBits().test(FeatureGFX90AInsts))
|
|
return 512;
|
|
if (!isGFX10Plus(*STI))
|
|
return 256;
|
|
bool IsWave32 = STI->getFeatureBits().test(FeatureWavefrontSize32);
|
|
if (STI->getFeatureBits().test(Feature1_5xVGPRs))
|
|
return IsWave32 ? 1536 : 768;
|
|
return IsWave32 ? 1024 : 512;
|
|
}
|
|
|
|
unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo *STI) { return 256; }
|
|
|
|
unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI,
|
|
unsigned DynamicVGPRBlockSize) {
|
|
if (STI->getFeatureBits().test(FeatureGFX90AInsts))
|
|
return 512;
|
|
|
|
// Temporarily check the subtarget feature, until we fully switch to using
|
|
// attributes.
|
|
if (DynamicVGPRBlockSize != 0 ||
|
|
STI->getFeatureBits().test(FeatureDynamicVGPR))
|
|
// On GFX12 we can allocate at most 8 blocks of VGPRs.
|
|
return 8 * getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
|
|
return getAddressableNumArchVGPRs(STI);
|
|
}
|
|
|
|
unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo *STI,
|
|
unsigned NumVGPRs,
|
|
unsigned DynamicVGPRBlockSize) {
|
|
return getNumWavesPerEUWithNumVGPRs(
|
|
NumVGPRs, getVGPRAllocGranule(STI, DynamicVGPRBlockSize),
|
|
getMaxWavesPerEU(STI), getTotalNumVGPRs(STI));
|
|
}
|
|
|
|
unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
|
|
unsigned MaxWaves,
|
|
unsigned TotalNumVGPRs) {
|
|
if (NumVGPRs < Granule)
|
|
return MaxWaves;
|
|
unsigned RoundedRegs = alignTo(NumVGPRs, Granule);
|
|
return std::min(std::max(TotalNumVGPRs / RoundedRegs, 1u), MaxWaves);
|
|
}
|
|
|
|
unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
|
|
AMDGPUSubtarget::Generation Gen) {
|
|
if (Gen >= AMDGPUSubtarget::GFX10)
|
|
return MaxWaves;
|
|
|
|
if (Gen >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
|
|
if (SGPRs <= 80)
|
|
return 10;
|
|
if (SGPRs <= 88)
|
|
return 9;
|
|
if (SGPRs <= 100)
|
|
return 8;
|
|
return 7;
|
|
}
|
|
if (SGPRs <= 48)
|
|
return 10;
|
|
if (SGPRs <= 56)
|
|
return 9;
|
|
if (SGPRs <= 64)
|
|
return 8;
|
|
if (SGPRs <= 72)
|
|
return 7;
|
|
if (SGPRs <= 80)
|
|
return 6;
|
|
return 5;
|
|
}
|
|
|
|
unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
|
|
unsigned DynamicVGPRBlockSize) {
|
|
assert(WavesPerEU != 0);
|
|
|
|
unsigned MaxWavesPerEU = getMaxWavesPerEU(STI);
|
|
if (WavesPerEU >= MaxWavesPerEU)
|
|
return 0;
|
|
|
|
unsigned TotNumVGPRs = getTotalNumVGPRs(STI);
|
|
unsigned AddrsableNumVGPRs =
|
|
getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
|
|
unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
|
|
unsigned MaxNumVGPRs = alignDown(TotNumVGPRs / WavesPerEU, Granule);
|
|
|
|
if (MaxNumVGPRs == alignDown(TotNumVGPRs / MaxWavesPerEU, Granule))
|
|
return 0;
|
|
|
|
unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, AddrsableNumVGPRs,
|
|
DynamicVGPRBlockSize);
|
|
if (WavesPerEU < MinWavesPerEU)
|
|
return getMinNumVGPRs(STI, MinWavesPerEU, DynamicVGPRBlockSize);
|
|
|
|
unsigned MaxNumVGPRsNext = alignDown(TotNumVGPRs / (WavesPerEU + 1), Granule);
|
|
unsigned MinNumVGPRs = 1 + std::min(MaxNumVGPRs - Granule, MaxNumVGPRsNext);
|
|
return std::min(MinNumVGPRs, AddrsableNumVGPRs);
|
|
}
|
|
|
|
unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
|
|
unsigned DynamicVGPRBlockSize) {
|
|
assert(WavesPerEU != 0);
|
|
|
|
unsigned MaxNumVGPRs =
|
|
alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
|
|
getVGPRAllocGranule(STI, DynamicVGPRBlockSize));
|
|
unsigned AddressableNumVGPRs =
|
|
getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
|
|
return std::min(MaxNumVGPRs, AddressableNumVGPRs);
|
|
}
|
|
|
|
unsigned getEncodedNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs,
|
|
std::optional<bool> EnableWavefrontSize32) {
|
|
return getGranulatedNumRegisterBlocks(
|
|
NumVGPRs, getVGPREncodingGranule(STI, EnableWavefrontSize32)) -
|
|
1;
|
|
}
|
|
|
|
unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo *STI,
|
|
unsigned NumVGPRs,
|
|
unsigned DynamicVGPRBlockSize,
|
|
std::optional<bool> EnableWavefrontSize32) {
|
|
return getGranulatedNumRegisterBlocks(
|
|
NumVGPRs,
|
|
getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
|
|
}
|
|
} // end namespace IsaInfo
|
|
|
|
void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,
|
|
const MCSubtargetInfo *STI) {
|
|
IsaVersion Version = getIsaVersion(STI->getCPU());
|
|
KernelCode.amd_kernel_code_version_major = 1;
|
|
KernelCode.amd_kernel_code_version_minor = 2;
|
|
KernelCode.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
|
|
KernelCode.amd_machine_version_major = Version.Major;
|
|
KernelCode.amd_machine_version_minor = Version.Minor;
|
|
KernelCode.amd_machine_version_stepping = Version.Stepping;
|
|
KernelCode.kernel_code_entry_byte_offset = sizeof(amd_kernel_code_t);
|
|
if (STI->getFeatureBits().test(FeatureWavefrontSize32)) {
|
|
KernelCode.wavefront_size = 5;
|
|
KernelCode.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
|
|
} else {
|
|
KernelCode.wavefront_size = 6;
|
|
}
|
|
|
|
// If the code object does not support indirect functions, then the value must
|
|
// be 0xffffffff.
|
|
KernelCode.call_convention = -1;
|
|
|
|
// These alignment values are specified in powers of two, so alignment =
|
|
// 2^n. The minimum alignment is 2^4 = 16.
|
|
KernelCode.kernarg_segment_alignment = 4;
|
|
KernelCode.group_segment_alignment = 4;
|
|
KernelCode.private_segment_alignment = 4;
|
|
|
|
if (Version.Major >= 10) {
|
|
KernelCode.compute_pgm_resource_registers |=
|
|
S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
|
|
S_00B848_MEM_ORDERED(1) | S_00B848_FWD_PROGRESS(1);
|
|
}
|
|
}
|
|
|
|
bool isGroupSegment(const GlobalValue *GV) {
|
|
return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
|
|
}
|
|
|
|
bool isGlobalSegment(const GlobalValue *GV) {
|
|
return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
|
|
}
|
|
|
|
bool isReadOnlySegment(const GlobalValue *GV) {
|
|
unsigned AS = GV->getAddressSpace();
|
|
return AS == AMDGPUAS::CONSTANT_ADDRESS ||
|
|
AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
|
|
}
|
|
|
|
bool shouldEmitConstantsToTextSection(const Triple &TT) {
|
|
return TT.getArch() == Triple::r600;
|
|
}
|
|
|
|
static bool isValidRegPrefix(char C) {
|
|
return C == 'v' || C == 's' || C == 'a';
|
|
}
|
|
|
|
std::tuple<char, unsigned, unsigned>
|
|
parseAsmConstraintPhysReg(StringRef Constraint) {
|
|
StringRef RegName = Constraint;
|
|
if (!RegName.consume_front("{") || !RegName.consume_back("}"))
|
|
return {};
|
|
|
|
char Kind = RegName.front();
|
|
if (!isValidRegPrefix(Kind))
|
|
return {};
|
|
|
|
RegName = RegName.drop_front();
|
|
if (RegName.consume_front("[")) {
|
|
unsigned Idx, End;
|
|
bool Failed = RegName.consumeInteger(10, Idx);
|
|
Failed |= !RegName.consume_front(":");
|
|
Failed |= RegName.consumeInteger(10, End);
|
|
Failed |= !RegName.consume_back("]");
|
|
if (!Failed) {
|
|
unsigned NumRegs = End - Idx + 1;
|
|
if (NumRegs > 1)
|
|
return {Kind, Idx, NumRegs};
|
|
}
|
|
} else {
|
|
unsigned Idx;
|
|
bool Failed = RegName.getAsInteger(10, Idx);
|
|
if (!Failed)
|
|
return {Kind, Idx, 1};
|
|
}
|
|
|
|
return {};
|
|
}
|
|
|
|
std::pair<unsigned, unsigned>
|
|
getIntegerPairAttribute(const Function &F, StringRef Name,
|
|
std::pair<unsigned, unsigned> Default,
|
|
bool OnlyFirstRequired) {
|
|
if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))
|
|
return {Attr->first, Attr->second.value_or(Default.second)};
|
|
return Default;
|
|
}
|
|
|
|
std::optional<std::pair<unsigned, std::optional<unsigned>>>
|
|
getIntegerPairAttribute(const Function &F, StringRef Name,
|
|
bool OnlyFirstRequired) {
|
|
Attribute A = F.getFnAttribute(Name);
|
|
if (!A.isStringAttribute())
|
|
return std::nullopt;
|
|
|
|
LLVMContext &Ctx = F.getContext();
|
|
std::pair<unsigned, std::optional<unsigned>> Ints;
|
|
std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
|
|
if (Strs.first.trim().getAsInteger(0, Ints.first)) {
|
|
Ctx.emitError("can't parse first integer attribute " + Name);
|
|
return std::nullopt;
|
|
}
|
|
unsigned Second = 0;
|
|
if (Strs.second.trim().getAsInteger(0, Second)) {
|
|
if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
|
|
Ctx.emitError("can't parse second integer attribute " + Name);
|
|
return std::nullopt;
|
|
}
|
|
} else {
|
|
Ints.second = Second;
|
|
}
|
|
|
|
return Ints;
|
|
}
|
|
|
|
SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
|
|
unsigned Size,
|
|
unsigned DefaultVal) {
|
|
std::optional<SmallVector<unsigned>> R =
|
|
getIntegerVecAttribute(F, Name, Size);
|
|
return R.has_value() ? *R : SmallVector<unsigned>(Size, DefaultVal);
|
|
}
|
|
|
|
std::optional<SmallVector<unsigned>>
|
|
getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {
|
|
assert(Size > 2);
|
|
LLVMContext &Ctx = F.getContext();
|
|
|
|
Attribute A = F.getFnAttribute(Name);
|
|
if (!A.isValid())
|
|
return std::nullopt;
|
|
if (!A.isStringAttribute()) {
|
|
Ctx.emitError(Name + " is not a string attribute");
|
|
return std::nullopt;
|
|
}
|
|
|
|
SmallVector<unsigned> Vals(Size);
|
|
|
|
StringRef S = A.getValueAsString();
|
|
unsigned i = 0;
|
|
for (; !S.empty() && i < Size; i++) {
|
|
std::pair<StringRef, StringRef> Strs = S.split(',');
|
|
unsigned IntVal;
|
|
if (Strs.first.trim().getAsInteger(0, IntVal)) {
|
|
Ctx.emitError("can't parse integer attribute " + Strs.first + " in " +
|
|
Name);
|
|
return std::nullopt;
|
|
}
|
|
Vals[i] = IntVal;
|
|
S = Strs.second;
|
|
}
|
|
|
|
if (!S.empty() || i < Size) {
|
|
Ctx.emitError("attribute " + Name +
|
|
" has incorrect number of integers; expected " +
|
|
llvm::utostr(Size));
|
|
return std::nullopt;
|
|
}
|
|
return Vals;
|
|
}
|
|
|
|
bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
|
|
assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");
|
|
for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {
|
|
auto Low =
|
|
mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 0))->getValue();
|
|
auto High =
|
|
mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 1))->getValue();
|
|
// There are two types of [A; B) ranges:
|
|
// A < B, e.g. [4; 5) which is a range that only includes 4.
|
|
// A > B, e.g. [5; 4) which is a range that wraps around and includes
|
|
// everything except 4.
|
|
if (Low.ult(High)) {
|
|
if (Low.ule(Val) && High.ugt(Val))
|
|
return true;
|
|
} else {
|
|
if (Low.uge(Val) && High.ult(Val))
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
unsigned getVmcntBitMask(const IsaVersion &Version) {
|
|
return (1 << (getVmcntBitWidthLo(Version.Major) +
|
|
getVmcntBitWidthHi(Version.Major))) -
|
|
1;
|
|
}
|
|
|
|
unsigned getLoadcntBitMask(const IsaVersion &Version) {
|
|
return (1 << getLoadcntBitWidth(Version.Major)) - 1;
|
|
}
|
|
|
|
unsigned getSamplecntBitMask(const IsaVersion &Version) {
|
|
return (1 << getSamplecntBitWidth(Version.Major)) - 1;
|
|
}
|
|
|
|
unsigned getBvhcntBitMask(const IsaVersion &Version) {
|
|
return (1 << getBvhcntBitWidth(Version.Major)) - 1;
|
|
}
|
|
|
|
unsigned getExpcntBitMask(const IsaVersion &Version) {
|
|
return (1 << getExpcntBitWidth(Version.Major)) - 1;
|
|
}
|
|
|
|
unsigned getLgkmcntBitMask(const IsaVersion &Version) {
|
|
return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
|
|
}
|
|
|
|
unsigned getDscntBitMask(const IsaVersion &Version) {
|
|
return (1 << getDscntBitWidth(Version.Major)) - 1;
|
|
}
|
|
|
|
unsigned getKmcntBitMask(const IsaVersion &Version) {
|
|
return (1 << getKmcntBitWidth(Version.Major)) - 1;
|
|
}
|
|
|
|
unsigned getXcntBitMask(const IsaVersion &Version) {
|
|
return (1 << getXcntBitWidth(Version.Major, Version.Minor)) - 1;
|
|
}
|
|
|
|
unsigned getStorecntBitMask(const IsaVersion &Version) {
|
|
return (1 << getStorecntBitWidth(Version.Major)) - 1;
|
|
}
|
|
|
|
unsigned getWaitcntBitMask(const IsaVersion &Version) {
|
|
unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(Version.Major),
|
|
getVmcntBitWidthLo(Version.Major));
|
|
unsigned Expcnt = getBitMask(getExpcntBitShift(Version.Major),
|
|
getExpcntBitWidth(Version.Major));
|
|
unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(Version.Major),
|
|
getLgkmcntBitWidth(Version.Major));
|
|
unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(Version.Major),
|
|
getVmcntBitWidthHi(Version.Major));
|
|
return VmcntLo | Expcnt | Lgkmcnt | VmcntHi;
|
|
}
|
|
|
|
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
|
|
unsigned VmcntLo = unpackBits(Waitcnt, getVmcntBitShiftLo(Version.Major),
|
|
getVmcntBitWidthLo(Version.Major));
|
|
unsigned VmcntHi = unpackBits(Waitcnt, getVmcntBitShiftHi(Version.Major),
|
|
getVmcntBitWidthHi(Version.Major));
|
|
return VmcntLo | VmcntHi << getVmcntBitWidthLo(Version.Major);
|
|
}
|
|
|
|
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
|
|
return unpackBits(Waitcnt, getExpcntBitShift(Version.Major),
|
|
getExpcntBitWidth(Version.Major));
|
|
}
|
|
|
|
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
|
|
return unpackBits(Waitcnt, getLgkmcntBitShift(Version.Major),
|
|
getLgkmcntBitWidth(Version.Major));
|
|
}
|
|
|
|
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
|
|
unsigned &Expcnt, unsigned &Lgkmcnt) {
|
|
Vmcnt = decodeVmcnt(Version, Waitcnt);
|
|
Expcnt = decodeExpcnt(Version, Waitcnt);
|
|
Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
|
|
}
|
|
|
|
Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
|
|
Waitcnt Decoded;
|
|
Decoded.LoadCnt = decodeVmcnt(Version, Encoded);
|
|
Decoded.ExpCnt = decodeExpcnt(Version, Encoded);
|
|
Decoded.DsCnt = decodeLgkmcnt(Version, Encoded);
|
|
return Decoded;
|
|
}
|
|
|
|
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
|
|
unsigned Vmcnt) {
|
|
Waitcnt = packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(Version.Major),
|
|
getVmcntBitWidthLo(Version.Major));
|
|
return packBits(Vmcnt >> getVmcntBitWidthLo(Version.Major), Waitcnt,
|
|
getVmcntBitShiftHi(Version.Major),
|
|
getVmcntBitWidthHi(Version.Major));
|
|
}
|
|
|
|
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
|
|
unsigned Expcnt) {
|
|
return packBits(Expcnt, Waitcnt, getExpcntBitShift(Version.Major),
|
|
getExpcntBitWidth(Version.Major));
|
|
}
|
|
|
|
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
|
|
unsigned Lgkmcnt) {
|
|
return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(Version.Major),
|
|
getLgkmcntBitWidth(Version.Major));
|
|
}
|
|
|
|
unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
|
|
unsigned Expcnt, unsigned Lgkmcnt) {
|
|
unsigned Waitcnt = getWaitcntBitMask(Version);
|
|
Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
|
|
Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
|
|
Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
|
|
return Waitcnt;
|
|
}
|
|
|
|
unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
|
|
return encodeWaitcnt(Version, Decoded.LoadCnt, Decoded.ExpCnt, Decoded.DsCnt);
|
|
}
|
|
|
|
static unsigned getCombinedCountBitMask(const IsaVersion &Version,
|
|
bool IsStore) {
|
|
unsigned Dscnt = getBitMask(getDscntBitShift(Version.Major),
|
|
getDscntBitWidth(Version.Major));
|
|
if (IsStore) {
|
|
unsigned Storecnt = getBitMask(getLoadcntStorecntBitShift(Version.Major),
|
|
getStorecntBitWidth(Version.Major));
|
|
return Dscnt | Storecnt;
|
|
}
|
|
unsigned Loadcnt = getBitMask(getLoadcntStorecntBitShift(Version.Major),
|
|
getLoadcntBitWidth(Version.Major));
|
|
return Dscnt | Loadcnt;
|
|
}
|
|
|
|
Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt) {
|
|
Waitcnt Decoded;
|
|
Decoded.LoadCnt =
|
|
unpackBits(LoadcntDscnt, getLoadcntStorecntBitShift(Version.Major),
|
|
getLoadcntBitWidth(Version.Major));
|
|
Decoded.DsCnt = unpackBits(LoadcntDscnt, getDscntBitShift(Version.Major),
|
|
getDscntBitWidth(Version.Major));
|
|
return Decoded;
|
|
}
|
|
|
|
Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt) {
|
|
Waitcnt Decoded;
|
|
Decoded.StoreCnt =
|
|
unpackBits(StorecntDscnt, getLoadcntStorecntBitShift(Version.Major),
|
|
getStorecntBitWidth(Version.Major));
|
|
Decoded.DsCnt = unpackBits(StorecntDscnt, getDscntBitShift(Version.Major),
|
|
getDscntBitWidth(Version.Major));
|
|
return Decoded;
|
|
}
|
|
|
|
static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,
|
|
unsigned Loadcnt) {
|
|
return packBits(Loadcnt, Waitcnt, getLoadcntStorecntBitShift(Version.Major),
|
|
getLoadcntBitWidth(Version.Major));
|
|
}
|
|
|
|
static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,
|
|
unsigned Storecnt) {
|
|
return packBits(Storecnt, Waitcnt, getLoadcntStorecntBitShift(Version.Major),
|
|
getStorecntBitWidth(Version.Major));
|
|
}
|
|
|
|
static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,
|
|
unsigned Dscnt) {
|
|
return packBits(Dscnt, Waitcnt, getDscntBitShift(Version.Major),
|
|
getDscntBitWidth(Version.Major));
|
|
}
|
|
|
|
static unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
|
|
unsigned Dscnt) {
|
|
unsigned Waitcnt = getCombinedCountBitMask(Version, false);
|
|
Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);
|
|
Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
|
|
return Waitcnt;
|
|
}
|
|
|
|
unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded) {
|
|
return encodeLoadcntDscnt(Version, Decoded.LoadCnt, Decoded.DsCnt);
|
|
}
|
|
|
|
static unsigned encodeStorecntDscnt(const IsaVersion &Version,
|
|
unsigned Storecnt, unsigned Dscnt) {
|
|
unsigned Waitcnt = getCombinedCountBitMask(Version, true);
|
|
Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);
|
|
Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
|
|
return Waitcnt;
|
|
}
|
|
|
|
unsigned encodeStorecntDscnt(const IsaVersion &Version,
|
|
const Waitcnt &Decoded) {
|
|
return encodeStorecntDscnt(Version, Decoded.StoreCnt, Decoded.DsCnt);
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Custom Operand Values
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
static unsigned getDefaultCustomOperandEncoding(const CustomOperandVal *Opr,
|
|
int Size,
|
|
const MCSubtargetInfo &STI) {
|
|
unsigned Enc = 0;
|
|
for (int Idx = 0; Idx < Size; ++Idx) {
|
|
const auto &Op = Opr[Idx];
|
|
if (Op.isSupported(STI))
|
|
Enc |= Op.encode(Op.Default);
|
|
}
|
|
return Enc;
|
|
}
|
|
|
|
static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr,
|
|
int Size, unsigned Code,
|
|
bool &HasNonDefaultVal,
|
|
const MCSubtargetInfo &STI) {
|
|
unsigned UsedOprMask = 0;
|
|
HasNonDefaultVal = false;
|
|
for (int Idx = 0; Idx < Size; ++Idx) {
|
|
const auto &Op = Opr[Idx];
|
|
if (!Op.isSupported(STI))
|
|
continue;
|
|
UsedOprMask |= Op.getMask();
|
|
unsigned Val = Op.decode(Code);
|
|
if (!Op.isValid(Val))
|
|
return false;
|
|
HasNonDefaultVal |= (Val != Op.Default);
|
|
}
|
|
return (Code & ~UsedOprMask) == 0;
|
|
}
|
|
|
|
static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size,
|
|
unsigned Code, int &Idx, StringRef &Name,
|
|
unsigned &Val, bool &IsDefault,
|
|
const MCSubtargetInfo &STI) {
|
|
while (Idx < Size) {
|
|
const auto &Op = Opr[Idx++];
|
|
if (Op.isSupported(STI)) {
|
|
Name = Op.Name;
|
|
Val = Op.decode(Code);
|
|
IsDefault = (Val == Op.Default);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static int encodeCustomOperandVal(const CustomOperandVal &Op,
|
|
int64_t InputVal) {
|
|
if (InputVal < 0 || InputVal > Op.Max)
|
|
return OPR_VAL_INVALID;
|
|
return Op.encode(InputVal);
|
|
}
|
|
|
|
static int encodeCustomOperand(const CustomOperandVal *Opr, int Size,
|
|
const StringRef Name, int64_t InputVal,
|
|
unsigned &UsedOprMask,
|
|
const MCSubtargetInfo &STI) {
|
|
int InvalidId = OPR_ID_UNKNOWN;
|
|
for (int Idx = 0; Idx < Size; ++Idx) {
|
|
const auto &Op = Opr[Idx];
|
|
if (Op.Name == Name) {
|
|
if (!Op.isSupported(STI)) {
|
|
InvalidId = OPR_ID_UNSUPPORTED;
|
|
continue;
|
|
}
|
|
auto OprMask = Op.getMask();
|
|
if (OprMask & UsedOprMask)
|
|
return OPR_ID_DUPLICATE;
|
|
UsedOprMask |= OprMask;
|
|
return encodeCustomOperandVal(Op, InputVal);
|
|
}
|
|
}
|
|
return InvalidId;
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// DepCtr
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace DepCtr {
|
|
|
|
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) {
|
|
static int Default = -1;
|
|
if (Default == -1)
|
|
Default = getDefaultCustomOperandEncoding(DepCtrInfo, DEP_CTR_SIZE, STI);
|
|
return Default;
|
|
}
|
|
|
|
bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
|
|
const MCSubtargetInfo &STI) {
|
|
return isSymbolicCustomOperandEncoding(DepCtrInfo, DEP_CTR_SIZE, Code,
|
|
HasNonDefaultVal, STI);
|
|
}
|
|
|
|
bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
|
|
bool &IsDefault, const MCSubtargetInfo &STI) {
|
|
return decodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Code, Id, Name, Val,
|
|
IsDefault, STI);
|
|
}
|
|
|
|
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
|
|
const MCSubtargetInfo &STI) {
|
|
return encodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Name, Val, UsedOprMask,
|
|
STI);
|
|
}
|
|
|
|
unsigned decodeFieldVmVsrc(unsigned Encoded) {
|
|
return unpackBits(Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());
|
|
}
|
|
|
|
unsigned decodeFieldVaVdst(unsigned Encoded) {
|
|
return unpackBits(Encoded, getVaVdstBitShift(), getVaVdstBitWidth());
|
|
}
|
|
|
|
unsigned decodeFieldSaSdst(unsigned Encoded) {
|
|
return unpackBits(Encoded, getSaSdstBitShift(), getSaSdstBitWidth());
|
|
}
|
|
|
|
unsigned decodeFieldVaSdst(unsigned Encoded) {
|
|
return unpackBits(Encoded, getVaSdstBitShift(), getVaSdstBitWidth());
|
|
}
|
|
|
|
unsigned decodeFieldVaVcc(unsigned Encoded) {
|
|
return unpackBits(Encoded, getVaVccBitShift(), getVaVccBitWidth());
|
|
}
|
|
|
|
unsigned decodeFieldVaSsrc(unsigned Encoded) {
|
|
return unpackBits(Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());
|
|
}
|
|
|
|
unsigned decodeFieldHoldCnt(unsigned Encoded) {
|
|
return unpackBits(Encoded, getHoldCntBitShift(), getHoldCntWidth());
|
|
}
|
|
|
|
unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
|
|
return packBits(VmVsrc, Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());
|
|
}
|
|
|
|
unsigned encodeFieldVmVsrc(unsigned VmVsrc) {
|
|
return encodeFieldVmVsrc(0xffff, VmVsrc);
|
|
}
|
|
|
|
unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {
|
|
return packBits(VaVdst, Encoded, getVaVdstBitShift(), getVaVdstBitWidth());
|
|
}
|
|
|
|
unsigned encodeFieldVaVdst(unsigned VaVdst) {
|
|
return encodeFieldVaVdst(0xffff, VaVdst);
|
|
}
|
|
|
|
unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {
|
|
return packBits(SaSdst, Encoded, getSaSdstBitShift(), getSaSdstBitWidth());
|
|
}
|
|
|
|
unsigned encodeFieldSaSdst(unsigned SaSdst) {
|
|
return encodeFieldSaSdst(0xffff, SaSdst);
|
|
}
|
|
|
|
unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
|
|
return packBits(VaSdst, Encoded, getVaSdstBitShift(), getVaSdstBitWidth());
|
|
}
|
|
|
|
unsigned encodeFieldVaSdst(unsigned VaSdst) {
|
|
return encodeFieldVaSdst(0xffff, VaSdst);
|
|
}
|
|
|
|
unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
|
|
return packBits(VaVcc, Encoded, getVaVccBitShift(), getVaVccBitWidth());
|
|
}
|
|
|
|
unsigned encodeFieldVaVcc(unsigned VaVcc) {
|
|
return encodeFieldVaVcc(0xffff, VaVcc);
|
|
}
|
|
|
|
unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
|
|
return packBits(VaSsrc, Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());
|
|
}
|
|
|
|
unsigned encodeFieldVaSsrc(unsigned VaSsrc) {
|
|
return encodeFieldVaSsrc(0xffff, VaSsrc);
|
|
}
|
|
|
|
unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt) {
|
|
return packBits(HoldCnt, Encoded, getHoldCntBitShift(), getHoldCntWidth());
|
|
}
|
|
|
|
unsigned encodeFieldHoldCnt(unsigned HoldCnt) {
|
|
return encodeFieldHoldCnt(0xffff, HoldCnt);
|
|
}
|
|
|
|
} // namespace DepCtr
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// exp tgt
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace Exp {
|
|
|
|
struct ExpTgt {
|
|
StringLiteral Name;
|
|
unsigned Tgt;
|
|
unsigned MaxIndex;
|
|
};
|
|
|
|
// clang-format off
|
|
static constexpr ExpTgt ExpTgtInfo[] = {
|
|
{{"null"}, ET_NULL, ET_NULL_MAX_IDX},
|
|
{{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX},
|
|
{{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX},
|
|
{{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX},
|
|
{{"pos"}, ET_POS0, ET_POS_MAX_IDX},
|
|
{{"dual_src_blend"},ET_DUAL_SRC_BLEND0, ET_DUAL_SRC_BLEND_MAX_IDX},
|
|
{{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX},
|
|
};
|
|
// clang-format on
|
|
|
|
bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
|
|
for (const ExpTgt &Val : ExpTgtInfo) {
|
|
if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
|
|
Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
|
|
Name = Val.Name;
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
unsigned getTgtId(const StringRef Name) {
|
|
|
|
for (const ExpTgt &Val : ExpTgtInfo) {
|
|
if (Val.MaxIndex == 0 && Name == Val.Name)
|
|
return Val.Tgt;
|
|
|
|
if (Val.MaxIndex > 0 && Name.starts_with(Val.Name)) {
|
|
StringRef Suffix = Name.drop_front(Val.Name.size());
|
|
|
|
unsigned Id;
|
|
if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex)
|
|
return ET_INVALID;
|
|
|
|
// Disable leading zeroes
|
|
if (Suffix.size() > 1 && Suffix[0] == '0')
|
|
return ET_INVALID;
|
|
|
|
return Val.Tgt + Id;
|
|
}
|
|
}
|
|
return ET_INVALID;
|
|
}
|
|
|
|
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
|
|
switch (Id) {
|
|
case ET_NULL:
|
|
return !isGFX11Plus(STI);
|
|
case ET_POS4:
|
|
case ET_PRIM:
|
|
return isGFX10Plus(STI);
|
|
case ET_DUAL_SRC_BLEND0:
|
|
case ET_DUAL_SRC_BLEND1:
|
|
return isGFX11Plus(STI);
|
|
default:
|
|
if (Id >= ET_PARAM0 && Id <= ET_PARAM31)
|
|
return !isGFX11Plus(STI);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
} // namespace Exp
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MTBUF Format
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace MTBUFFormat {
|
|
|
|
int64_t getDfmt(const StringRef Name) {
|
|
for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
|
|
if (Name == DfmtSymbolic[Id])
|
|
return Id;
|
|
}
|
|
return DFMT_UNDEF;
|
|
}
|
|
|
|
StringRef getDfmtName(unsigned Id) {
|
|
assert(Id <= DFMT_MAX);
|
|
return DfmtSymbolic[Id];
|
|
}
|
|
|
|
static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
|
|
if (isSI(STI) || isCI(STI))
|
|
return NfmtSymbolicSICI;
|
|
if (isVI(STI) || isGFX9(STI))
|
|
return NfmtSymbolicVI;
|
|
return NfmtSymbolicGFX10;
|
|
}
|
|
|
|
int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
|
|
const auto *lookupTable = getNfmtLookupTable(STI);
|
|
for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
|
|
if (Name == lookupTable[Id])
|
|
return Id;
|
|
}
|
|
return NFMT_UNDEF;
|
|
}
|
|
|
|
StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
|
|
assert(Id <= NFMT_MAX);
|
|
return getNfmtLookupTable(STI)[Id];
|
|
}
|
|
|
|
bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
|
|
unsigned Dfmt;
|
|
unsigned Nfmt;
|
|
decodeDfmtNfmt(Id, Dfmt, Nfmt);
|
|
return isValidNfmt(Nfmt, STI);
|
|
}
|
|
|
|
bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
|
|
return !getNfmtName(Id, STI).empty();
|
|
}
|
|
|
|
int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
|
|
return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
|
|
}
|
|
|
|
void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
|
|
Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
|
|
Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
|
|
}
|
|
|
|
int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {
|
|
if (isGFX11Plus(STI)) {
|
|
for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
|
|
if (Name == UfmtSymbolicGFX11[Id])
|
|
return Id;
|
|
}
|
|
} else {
|
|
for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
|
|
if (Name == UfmtSymbolicGFX10[Id])
|
|
return Id;
|
|
}
|
|
}
|
|
return UFMT_UNDEF;
|
|
}
|
|
|
|
StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) {
|
|
if (isValidUnifiedFormat(Id, STI))
|
|
return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];
|
|
return "";
|
|
}
|
|
|
|
bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {
|
|
return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;
|
|
}
|
|
|
|
int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
|
|
const MCSubtargetInfo &STI) {
|
|
int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
|
|
if (isGFX11Plus(STI)) {
|
|
for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
|
|
if (Fmt == DfmtNfmt2UFmtGFX11[Id])
|
|
return Id;
|
|
}
|
|
} else {
|
|
for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
|
|
if (Fmt == DfmtNfmt2UFmtGFX10[Id])
|
|
return Id;
|
|
}
|
|
}
|
|
return UFMT_UNDEF;
|
|
}
|
|
|
|
bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
|
|
return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
|
|
}
|
|
|
|
unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
|
|
if (isGFX10Plus(STI))
|
|
return UFMT_DEFAULT;
|
|
return DFMT_NFMT_DEFAULT;
|
|
}
|
|
|
|
} // namespace MTBUFFormat
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// SendMsg
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace SendMsg {
|
|
|
|
static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
|
|
return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
|
|
}
|
|
|
|
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
|
|
return (MsgId & ~(getMsgIdMask(STI))) == 0;
|
|
}
|
|
|
|
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
|
|
bool Strict) {
|
|
assert(isValidMsgId(MsgId, STI));
|
|
|
|
if (!Strict)
|
|
return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
|
|
|
|
if (msgRequiresOp(MsgId, STI)) {
|
|
if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
|
|
return false;
|
|
|
|
return !getMsgOpName(MsgId, OpId, STI).empty();
|
|
}
|
|
|
|
return OpId == OP_NONE_;
|
|
}
|
|
|
|
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
|
|
const MCSubtargetInfo &STI, bool Strict) {
|
|
assert(isValidMsgOp(MsgId, OpId, STI, Strict));
|
|
|
|
if (!Strict)
|
|
return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);
|
|
|
|
if (!isGFX11Plus(STI)) {
|
|
switch (MsgId) {
|
|
case ID_GS_PreGFX11:
|
|
return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
|
|
case ID_GS_DONE_PreGFX11:
|
|
return (OpId == OP_GS_NOP)
|
|
? (StreamId == STREAM_ID_NONE_)
|
|
: (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
|
|
}
|
|
}
|
|
return StreamId == STREAM_ID_NONE_;
|
|
}
|
|
|
|
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
|
|
return MsgId == ID_SYSMSG ||
|
|
(!isGFX11Plus(STI) &&
|
|
(MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));
|
|
}
|
|
|
|
bool msgSupportsStream(int64_t MsgId, int64_t OpId,
|
|
const MCSubtargetInfo &STI) {
|
|
return !isGFX11Plus(STI) &&
|
|
(MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&
|
|
OpId != OP_GS_NOP;
|
|
}
|
|
|
|
void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
|
|
uint16_t &StreamId, const MCSubtargetInfo &STI) {
|
|
MsgId = Val & getMsgIdMask(STI);
|
|
if (isGFX11Plus(STI)) {
|
|
OpId = 0;
|
|
StreamId = 0;
|
|
} else {
|
|
OpId = (Val & OP_MASK_) >> OP_SHIFT_;
|
|
StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
|
|
}
|
|
}
|
|
|
|
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) {
|
|
return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_);
|
|
}
|
|
|
|
} // namespace SendMsg
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
unsigned getInitialPSInputAddr(const Function &F) {
|
|
return F.getFnAttributeAsParsedInteger("InitialPSInputAddr", 0);
|
|
}
|
|
|
|
bool getHasColorExport(const Function &F) {
|
|
// As a safe default always respond as if PS has color exports.
|
|
return F.getFnAttributeAsParsedInteger(
|
|
"amdgpu-color-export",
|
|
F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
|
|
}
|
|
|
|
bool getHasDepthExport(const Function &F) {
|
|
return F.getFnAttributeAsParsedInteger("amdgpu-depth-export", 0) != 0;
|
|
}
|
|
|
|
unsigned getDynamicVGPRBlockSize(const Function &F) {
|
|
unsigned BlockSize =
|
|
F.getFnAttributeAsParsedInteger("amdgpu-dynamic-vgpr-block-size", 0);
|
|
|
|
if (BlockSize == 16 || BlockSize == 32)
|
|
return BlockSize;
|
|
|
|
return 0;
|
|
}
|
|
|
|
bool hasXNACK(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureXNACK);
|
|
}
|
|
|
|
bool hasSRAMECC(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureSRAMECC);
|
|
}
|
|
|
|
bool hasMIMG_R128(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureMIMG_R128) &&
|
|
!STI.hasFeature(AMDGPU::FeatureR128A16);
|
|
}
|
|
|
|
bool hasA16(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureA16);
|
|
}
|
|
|
|
bool hasG16(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureG16);
|
|
}
|
|
|
|
bool hasPackedD16(const MCSubtargetInfo &STI) {
|
|
return !STI.hasFeature(AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
|
|
!isSI(STI);
|
|
}
|
|
|
|
bool hasGDS(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGDS);
|
|
}
|
|
|
|
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
|
|
auto Version = getIsaVersion(STI.getCPU());
|
|
if (Version.Major == 10)
|
|
return Version.Minor >= 3 ? 13 : 5;
|
|
if (Version.Major == 11)
|
|
return 5;
|
|
if (Version.Major >= 12)
|
|
return HasSampler ? 4 : 5;
|
|
return 0;
|
|
}
|
|
|
|
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {
|
|
if (isGFX1250(STI))
|
|
return 32;
|
|
return 16;
|
|
}
|
|
|
|
bool isSI(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureSouthernIslands);
|
|
}
|
|
|
|
bool isCI(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureSeaIslands);
|
|
}
|
|
|
|
bool isVI(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureVolcanicIslands);
|
|
}
|
|
|
|
bool isGFX9(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGFX9);
|
|
}
|
|
|
|
bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
|
|
return isGFX9(STI) || isGFX10(STI);
|
|
}
|
|
|
|
bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI) {
|
|
return isGFX9(STI) || isGFX10(STI) || isGFX11(STI);
|
|
}
|
|
|
|
bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) {
|
|
return isVI(STI) || isGFX9(STI) || isGFX10(STI);
|
|
}
|
|
|
|
bool isGFX8Plus(const MCSubtargetInfo &STI) {
|
|
return isVI(STI) || isGFX9Plus(STI);
|
|
}
|
|
|
|
bool isGFX9Plus(const MCSubtargetInfo &STI) {
|
|
return isGFX9(STI) || isGFX10Plus(STI);
|
|
}
|
|
|
|
bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }
|
|
|
|
bool isGFX10(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGFX10);
|
|
}
|
|
|
|
bool isGFX10_GFX11(const MCSubtargetInfo &STI) {
|
|
return isGFX10(STI) || isGFX11(STI);
|
|
}
|
|
|
|
bool isGFX10Plus(const MCSubtargetInfo &STI) {
|
|
return isGFX10(STI) || isGFX11Plus(STI);
|
|
}
|
|
|
|
bool isGFX11(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGFX11);
|
|
}
|
|
|
|
bool isGFX11Plus(const MCSubtargetInfo &STI) {
|
|
return isGFX11(STI) || isGFX12Plus(STI);
|
|
}
|
|
|
|
bool isGFX12(const MCSubtargetInfo &STI) {
|
|
return STI.getFeatureBits()[AMDGPU::FeatureGFX12];
|
|
}
|
|
|
|
bool isGFX12Plus(const MCSubtargetInfo &STI) { return isGFX12(STI); }
|
|
|
|
bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }
|
|
|
|
bool isGFX1250(const MCSubtargetInfo &STI) {
|
|
return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
|
|
}
|
|
|
|
bool supportsWGP(const MCSubtargetInfo &STI) {
|
|
if (isGFX1250(STI))
|
|
return false;
|
|
return isGFX10Plus(STI);
|
|
}
|
|
|
|
bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }
|
|
|
|
bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
|
|
return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
|
|
}
|
|
|
|
bool isGFX10Before1030(const MCSubtargetInfo &STI) {
|
|
return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
|
|
}
|
|
|
|
bool isGCN3Encoding(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGCN3Encoding);
|
|
}
|
|
|
|
bool isGFX10_AEncoding(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGFX10_AEncoding);
|
|
}
|
|
|
|
bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGFX10_BEncoding);
|
|
}
|
|
|
|
bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGFX10_3Insts);
|
|
}
|
|
|
|
bool isGFX10_3_GFX11(const MCSubtargetInfo &STI) {
|
|
return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);
|
|
}
|
|
|
|
bool isGFX90A(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGFX90AInsts);
|
|
}
|
|
|
|
bool isGFX940(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureGFX940Insts);
|
|
}
|
|
|
|
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureArchitectedFlatScratch);
|
|
}
|
|
|
|
bool hasMAIInsts(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureMAIInsts);
|
|
}
|
|
|
|
bool hasVOPD(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureVOPD);
|
|
}
|
|
|
|
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureDPPSrc1SGPR);
|
|
}
|
|
|
|
unsigned hasKernargPreload(const MCSubtargetInfo &STI) {
|
|
return STI.hasFeature(AMDGPU::FeatureKernargPreload);
|
|
}
|
|
|
|
int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,
|
|
int32_t ArgNumVGPR) {
|
|
if (has90AInsts && ArgNumAGPR)
|
|
return alignTo(ArgNumVGPR, 4) + ArgNumAGPR;
|
|
return std::max(ArgNumVGPR, ArgNumAGPR);
|
|
}
|
|
|
|
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {
|
|
const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
|
|
const MCRegister FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0);
|
|
return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
|
|
Reg == AMDGPU::SCC;
|
|
}
|
|
|
|
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {
|
|
return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
|
|
}
|
|
|
|
#define MAP_REG2REG \
|
|
using namespace AMDGPU; \
|
|
switch (Reg.id()) { \
|
|
default: \
|
|
return Reg; \
|
|
CASE_CI_VI(FLAT_SCR) \
|
|
CASE_CI_VI(FLAT_SCR_LO) \
|
|
CASE_CI_VI(FLAT_SCR_HI) \
|
|
CASE_VI_GFX9PLUS(TTMP0) \
|
|
CASE_VI_GFX9PLUS(TTMP1) \
|
|
CASE_VI_GFX9PLUS(TTMP2) \
|
|
CASE_VI_GFX9PLUS(TTMP3) \
|
|
CASE_VI_GFX9PLUS(TTMP4) \
|
|
CASE_VI_GFX9PLUS(TTMP5) \
|
|
CASE_VI_GFX9PLUS(TTMP6) \
|
|
CASE_VI_GFX9PLUS(TTMP7) \
|
|
CASE_VI_GFX9PLUS(TTMP8) \
|
|
CASE_VI_GFX9PLUS(TTMP9) \
|
|
CASE_VI_GFX9PLUS(TTMP10) \
|
|
CASE_VI_GFX9PLUS(TTMP11) \
|
|
CASE_VI_GFX9PLUS(TTMP12) \
|
|
CASE_VI_GFX9PLUS(TTMP13) \
|
|
CASE_VI_GFX9PLUS(TTMP14) \
|
|
CASE_VI_GFX9PLUS(TTMP15) \
|
|
CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
|
|
CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
|
|
CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
|
|
CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
|
|
CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
|
|
CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
|
|
CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
|
|
CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
|
|
CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
|
|
CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
|
|
CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
|
|
CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
|
|
CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
|
|
CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
|
|
CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
|
|
CASE_VI_GFX9PLUS( \
|
|
TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
|
|
CASE_GFXPRE11_GFX11PLUS(M0) \
|
|
CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \
|
|
CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \
|
|
}
|
|
|
|
#define CASE_CI_VI(node) \
|
|
assert(!isSI(STI)); \
|
|
case node: \
|
|
return isCI(STI) ? node##_ci : node##_vi;
|
|
|
|
#define CASE_VI_GFX9PLUS(node) \
|
|
case node: \
|
|
return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
|
|
|
|
#define CASE_GFXPRE11_GFX11PLUS(node) \
|
|
case node: \
|
|
return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;
|
|
|
|
#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \
|
|
case node: \
|
|
return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;
|
|
|
|
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI) {
|
|
if (STI.getTargetTriple().getArch() == Triple::r600)
|
|
return Reg;
|
|
MAP_REG2REG
|
|
}
|
|
|
|
#undef CASE_CI_VI
|
|
#undef CASE_VI_GFX9PLUS
|
|
#undef CASE_GFXPRE11_GFX11PLUS
|
|
#undef CASE_GFXPRE11_GFX11PLUS_TO
|
|
|
|
#define CASE_CI_VI(node) \
|
|
case node##_ci: \
|
|
case node##_vi: \
|
|
return node;
|
|
#define CASE_VI_GFX9PLUS(node) \
|
|
case node##_vi: \
|
|
case node##_gfx9plus: \
|
|
return node;
|
|
#define CASE_GFXPRE11_GFX11PLUS(node) \
|
|
case node##_gfx11plus: \
|
|
case node##_gfxpre11: \
|
|
return node;
|
|
#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)
|
|
|
|
MCRegister mc2PseudoReg(MCRegister Reg) { MAP_REG2REG }
|
|
|
|
bool isInlineValue(unsigned Reg) {
|
|
switch (Reg) {
|
|
case AMDGPU::SRC_SHARED_BASE_LO:
|
|
case AMDGPU::SRC_SHARED_BASE:
|
|
case AMDGPU::SRC_SHARED_LIMIT_LO:
|
|
case AMDGPU::SRC_SHARED_LIMIT:
|
|
case AMDGPU::SRC_PRIVATE_BASE_LO:
|
|
case AMDGPU::SRC_PRIVATE_BASE:
|
|
case AMDGPU::SRC_PRIVATE_LIMIT_LO:
|
|
case AMDGPU::SRC_PRIVATE_LIMIT:
|
|
case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
|
|
case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
|
|
case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
|
|
return true;
|
|
case AMDGPU::SRC_VCCZ:
|
|
case AMDGPU::SRC_EXECZ:
|
|
case AMDGPU::SRC_SCC:
|
|
return true;
|
|
case AMDGPU::SGPR_NULL:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
#undef CASE_CI_VI
|
|
#undef CASE_VI_GFX9PLUS
|
|
#undef CASE_GFXPRE11_GFX11PLUS
|
|
#undef CASE_GFXPRE11_GFX11PLUS_TO
|
|
#undef MAP_REG2REG
|
|
|
|
bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
|
|
assert(OpNo < Desc.NumOperands);
|
|
unsigned OpType = Desc.operands()[OpNo].OperandType;
|
|
return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
|
|
OpType <= AMDGPU::OPERAND_SRC_LAST;
|
|
}
|
|
|
|
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {
|
|
assert(OpNo < Desc.NumOperands);
|
|
unsigned OpType = Desc.operands()[OpNo].OperandType;
|
|
return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&
|
|
OpType <= AMDGPU::OPERAND_KIMM_LAST;
|
|
}
|
|
|
|
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
|
|
assert(OpNo < Desc.NumOperands);
|
|
unsigned OpType = Desc.operands()[OpNo].OperandType;
|
|
switch (OpType) {
|
|
case AMDGPU::OPERAND_REG_IMM_FP32:
|
|
case AMDGPU::OPERAND_REG_IMM_FP64:
|
|
case AMDGPU::OPERAND_REG_IMM_FP16:
|
|
case AMDGPU::OPERAND_REG_IMM_V2FP16:
|
|
case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
|
|
case AMDGPU::OPERAND_REG_INLINE_C_FP32:
|
|
case AMDGPU::OPERAND_REG_INLINE_C_FP64:
|
|
case AMDGPU::OPERAND_REG_INLINE_C_FP16:
|
|
case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
|
|
case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
|
|
case AMDGPU::OPERAND_REG_IMM_V2FP32:
|
|
case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
|
|
assert(OpNo < Desc.NumOperands);
|
|
unsigned OpType = Desc.operands()[OpNo].OperandType;
|
|
return (OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
|
|
OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST) ||
|
|
(OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST &&
|
|
OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST);
|
|
}
|
|
|
|
// Avoid using MCRegisterClass::getSize, since that function will go away
|
|
// (move from MC* level to Target* level). Return size in bits.
|
|
unsigned getRegBitWidth(unsigned RCID) {
|
|
switch (RCID) {
|
|
case AMDGPU::VGPR_16RegClassID:
|
|
case AMDGPU::VGPR_16_Lo128RegClassID:
|
|
case AMDGPU::SGPR_LO16RegClassID:
|
|
case AMDGPU::AGPR_LO16RegClassID:
|
|
return 16;
|
|
case AMDGPU::SGPR_32RegClassID:
|
|
case AMDGPU::VGPR_32RegClassID:
|
|
case AMDGPU::VRegOrLds_32RegClassID:
|
|
case AMDGPU::AGPR_32RegClassID:
|
|
case AMDGPU::VS_32RegClassID:
|
|
case AMDGPU::AV_32RegClassID:
|
|
case AMDGPU::SReg_32RegClassID:
|
|
case AMDGPU::SReg_32_XM0RegClassID:
|
|
case AMDGPU::SRegOrLds_32RegClassID:
|
|
return 32;
|
|
case AMDGPU::SGPR_64RegClassID:
|
|
case AMDGPU::VS_64RegClassID:
|
|
case AMDGPU::SReg_64RegClassID:
|
|
case AMDGPU::VReg_64RegClassID:
|
|
case AMDGPU::AReg_64RegClassID:
|
|
case AMDGPU::SReg_64_XEXECRegClassID:
|
|
case AMDGPU::VReg_64_Align2RegClassID:
|
|
case AMDGPU::AReg_64_Align2RegClassID:
|
|
case AMDGPU::AV_64RegClassID:
|
|
case AMDGPU::AV_64_Align2RegClassID:
|
|
return 64;
|
|
case AMDGPU::SGPR_96RegClassID:
|
|
case AMDGPU::SReg_96RegClassID:
|
|
case AMDGPU::VReg_96RegClassID:
|
|
case AMDGPU::AReg_96RegClassID:
|
|
case AMDGPU::VReg_96_Align2RegClassID:
|
|
case AMDGPU::AReg_96_Align2RegClassID:
|
|
case AMDGPU::AV_96RegClassID:
|
|
case AMDGPU::AV_96_Align2RegClassID:
|
|
return 96;
|
|
case AMDGPU::SGPR_128RegClassID:
|
|
case AMDGPU::SReg_128RegClassID:
|
|
case AMDGPU::VReg_128RegClassID:
|
|
case AMDGPU::AReg_128RegClassID:
|
|
case AMDGPU::VReg_128_Align2RegClassID:
|
|
case AMDGPU::AReg_128_Align2RegClassID:
|
|
case AMDGPU::AV_128RegClassID:
|
|
case AMDGPU::AV_128_Align2RegClassID:
|
|
case AMDGPU::SReg_128_XNULLRegClassID:
|
|
return 128;
|
|
case AMDGPU::SGPR_160RegClassID:
|
|
case AMDGPU::SReg_160RegClassID:
|
|
case AMDGPU::VReg_160RegClassID:
|
|
case AMDGPU::AReg_160RegClassID:
|
|
case AMDGPU::VReg_160_Align2RegClassID:
|
|
case AMDGPU::AReg_160_Align2RegClassID:
|
|
case AMDGPU::AV_160RegClassID:
|
|
case AMDGPU::AV_160_Align2RegClassID:
|
|
return 160;
|
|
case AMDGPU::SGPR_192RegClassID:
|
|
case AMDGPU::SReg_192RegClassID:
|
|
case AMDGPU::VReg_192RegClassID:
|
|
case AMDGPU::AReg_192RegClassID:
|
|
case AMDGPU::VReg_192_Align2RegClassID:
|
|
case AMDGPU::AReg_192_Align2RegClassID:
|
|
case AMDGPU::AV_192RegClassID:
|
|
case AMDGPU::AV_192_Align2RegClassID:
|
|
return 192;
|
|
case AMDGPU::SGPR_224RegClassID:
|
|
case AMDGPU::SReg_224RegClassID:
|
|
case AMDGPU::VReg_224RegClassID:
|
|
case AMDGPU::AReg_224RegClassID:
|
|
case AMDGPU::VReg_224_Align2RegClassID:
|
|
case AMDGPU::AReg_224_Align2RegClassID:
|
|
case AMDGPU::AV_224RegClassID:
|
|
case AMDGPU::AV_224_Align2RegClassID:
|
|
return 224;
|
|
case AMDGPU::SGPR_256RegClassID:
|
|
case AMDGPU::SReg_256RegClassID:
|
|
case AMDGPU::VReg_256RegClassID:
|
|
case AMDGPU::AReg_256RegClassID:
|
|
case AMDGPU::VReg_256_Align2RegClassID:
|
|
case AMDGPU::AReg_256_Align2RegClassID:
|
|
case AMDGPU::AV_256RegClassID:
|
|
case AMDGPU::AV_256_Align2RegClassID:
|
|
case AMDGPU::SReg_256_XNULLRegClassID:
|
|
return 256;
|
|
case AMDGPU::SGPR_288RegClassID:
|
|
case AMDGPU::SReg_288RegClassID:
|
|
case AMDGPU::VReg_288RegClassID:
|
|
case AMDGPU::AReg_288RegClassID:
|
|
case AMDGPU::VReg_288_Align2RegClassID:
|
|
case AMDGPU::AReg_288_Align2RegClassID:
|
|
case AMDGPU::AV_288RegClassID:
|
|
case AMDGPU::AV_288_Align2RegClassID:
|
|
return 288;
|
|
case AMDGPU::SGPR_320RegClassID:
|
|
case AMDGPU::SReg_320RegClassID:
|
|
case AMDGPU::VReg_320RegClassID:
|
|
case AMDGPU::AReg_320RegClassID:
|
|
case AMDGPU::VReg_320_Align2RegClassID:
|
|
case AMDGPU::AReg_320_Align2RegClassID:
|
|
case AMDGPU::AV_320RegClassID:
|
|
case AMDGPU::AV_320_Align2RegClassID:
|
|
return 320;
|
|
case AMDGPU::SGPR_352RegClassID:
|
|
case AMDGPU::SReg_352RegClassID:
|
|
case AMDGPU::VReg_352RegClassID:
|
|
case AMDGPU::AReg_352RegClassID:
|
|
case AMDGPU::VReg_352_Align2RegClassID:
|
|
case AMDGPU::AReg_352_Align2RegClassID:
|
|
case AMDGPU::AV_352RegClassID:
|
|
case AMDGPU::AV_352_Align2RegClassID:
|
|
return 352;
|
|
case AMDGPU::SGPR_384RegClassID:
|
|
case AMDGPU::SReg_384RegClassID:
|
|
case AMDGPU::VReg_384RegClassID:
|
|
case AMDGPU::AReg_384RegClassID:
|
|
case AMDGPU::VReg_384_Align2RegClassID:
|
|
case AMDGPU::AReg_384_Align2RegClassID:
|
|
case AMDGPU::AV_384RegClassID:
|
|
case AMDGPU::AV_384_Align2RegClassID:
|
|
return 384;
|
|
case AMDGPU::SGPR_512RegClassID:
|
|
case AMDGPU::SReg_512RegClassID:
|
|
case AMDGPU::VReg_512RegClassID:
|
|
case AMDGPU::AReg_512RegClassID:
|
|
case AMDGPU::VReg_512_Align2RegClassID:
|
|
case AMDGPU::AReg_512_Align2RegClassID:
|
|
case AMDGPU::AV_512RegClassID:
|
|
case AMDGPU::AV_512_Align2RegClassID:
|
|
return 512;
|
|
case AMDGPU::SGPR_1024RegClassID:
|
|
case AMDGPU::SReg_1024RegClassID:
|
|
case AMDGPU::VReg_1024RegClassID:
|
|
case AMDGPU::AReg_1024RegClassID:
|
|
case AMDGPU::VReg_1024_Align2RegClassID:
|
|
case AMDGPU::AReg_1024_Align2RegClassID:
|
|
case AMDGPU::AV_1024RegClassID:
|
|
case AMDGPU::AV_1024_Align2RegClassID:
|
|
return 1024;
|
|
default:
|
|
llvm_unreachable("Unexpected register class");
|
|
}
|
|
}
|
|
|
|
unsigned getRegBitWidth(const MCRegisterClass &RC) {
|
|
return getRegBitWidth(RC.getID());
|
|
}
|
|
|
|
unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc,
|
|
unsigned OpNo) {
|
|
assert(OpNo < Desc.NumOperands);
|
|
unsigned RCID = Desc.operands()[OpNo].RegClass;
|
|
return getRegBitWidth(RCID) / 8;
|
|
}
|
|
|
|
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
|
|
if (isInlinableIntLiteral(Literal))
|
|
return true;
|
|
|
|
uint64_t Val = static_cast<uint64_t>(Literal);
|
|
return (Val == llvm::bit_cast<uint64_t>(0.0)) ||
|
|
(Val == llvm::bit_cast<uint64_t>(1.0)) ||
|
|
(Val == llvm::bit_cast<uint64_t>(-1.0)) ||
|
|
(Val == llvm::bit_cast<uint64_t>(0.5)) ||
|
|
(Val == llvm::bit_cast<uint64_t>(-0.5)) ||
|
|
(Val == llvm::bit_cast<uint64_t>(2.0)) ||
|
|
(Val == llvm::bit_cast<uint64_t>(-2.0)) ||
|
|
(Val == llvm::bit_cast<uint64_t>(4.0)) ||
|
|
(Val == llvm::bit_cast<uint64_t>(-4.0)) ||
|
|
(Val == 0x3fc45f306dc9c882 && HasInv2Pi);
|
|
}
|
|
|
|
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
|
|
if (isInlinableIntLiteral(Literal))
|
|
return true;
|
|
|
|
// The actual type of the operand does not seem to matter as long
|
|
// as the bits match one of the inline immediate values. For example:
|
|
//
|
|
// -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
|
|
// so it is a legal inline immediate.
|
|
//
|
|
// 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
|
|
// floating-point, so it is a legal inline immediate.
|
|
|
|
uint32_t Val = static_cast<uint32_t>(Literal);
|
|
return (Val == llvm::bit_cast<uint32_t>(0.0f)) ||
|
|
(Val == llvm::bit_cast<uint32_t>(1.0f)) ||
|
|
(Val == llvm::bit_cast<uint32_t>(-1.0f)) ||
|
|
(Val == llvm::bit_cast<uint32_t>(0.5f)) ||
|
|
(Val == llvm::bit_cast<uint32_t>(-0.5f)) ||
|
|
(Val == llvm::bit_cast<uint32_t>(2.0f)) ||
|
|
(Val == llvm::bit_cast<uint32_t>(-2.0f)) ||
|
|
(Val == llvm::bit_cast<uint32_t>(4.0f)) ||
|
|
(Val == llvm::bit_cast<uint32_t>(-4.0f)) ||
|
|
(Val == 0x3e22f983 && HasInv2Pi);
|
|
}
|
|
|
|
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {
|
|
if (!HasInv2Pi)
|
|
return false;
|
|
if (isInlinableIntLiteral(Literal))
|
|
return true;
|
|
uint16_t Val = static_cast<uint16_t>(Literal);
|
|
return Val == 0x3F00 || // 0.5
|
|
Val == 0xBF00 || // -0.5
|
|
Val == 0x3F80 || // 1.0
|
|
Val == 0xBF80 || // -1.0
|
|
Val == 0x4000 || // 2.0
|
|
Val == 0xC000 || // -2.0
|
|
Val == 0x4080 || // 4.0
|
|
Val == 0xC080 || // -4.0
|
|
Val == 0x3E22; // 1.0 / (2.0 * pi)
|
|
}
|
|
|
|
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
|
|
return isInlinableLiteral32(Literal, HasInv2Pi);
|
|
}
|
|
|
|
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
|
|
if (!HasInv2Pi)
|
|
return false;
|
|
if (isInlinableIntLiteral(Literal))
|
|
return true;
|
|
uint16_t Val = static_cast<uint16_t>(Literal);
|
|
return Val == 0x3C00 || // 1.0
|
|
Val == 0xBC00 || // -1.0
|
|
Val == 0x3800 || // 0.5
|
|
Val == 0xB800 || // -0.5
|
|
Val == 0x4000 || // 2.0
|
|
Val == 0xC000 || // -2.0
|
|
Val == 0x4400 || // 4.0
|
|
Val == 0xC400 || // -4.0
|
|
Val == 0x3118; // 1/2pi
|
|
}
|
|
|
|
std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {
|
|
// Unfortunately, the Instruction Set Architecture Reference Guide is
|
|
// misleading about how the inline operands work for (packed) 16-bit
|
|
// instructions. In a nutshell, the actual HW behavior is:
|
|
//
|
|
// - integer encodings (-16 .. 64) are always produced as sign-extended
|
|
// 32-bit values
|
|
// - float encodings are produced as:
|
|
// - for F16 instructions: corresponding half-precision float values in
|
|
// the LSBs, 0 in the MSBs
|
|
// - for UI16 instructions: corresponding single-precision float value
|
|
int32_t Signed = static_cast<int32_t>(Literal);
|
|
if (Signed >= 0 && Signed <= 64)
|
|
return 128 + Signed;
|
|
|
|
if (Signed >= -16 && Signed <= -1)
|
|
return 192 + std::abs(Signed);
|
|
|
|
if (IsFloat) {
|
|
// clang-format off
|
|
switch (Literal) {
|
|
case 0x3800: return 240; // 0.5
|
|
case 0xB800: return 241; // -0.5
|
|
case 0x3C00: return 242; // 1.0
|
|
case 0xBC00: return 243; // -1.0
|
|
case 0x4000: return 244; // 2.0
|
|
case 0xC000: return 245; // -2.0
|
|
case 0x4400: return 246; // 4.0
|
|
case 0xC400: return 247; // -4.0
|
|
case 0x3118: return 248; // 1.0 / (2.0 * pi)
|
|
default: break;
|
|
}
|
|
// clang-format on
|
|
} else {
|
|
// clang-format off
|
|
switch (Literal) {
|
|
case 0x3F000000: return 240; // 0.5
|
|
case 0xBF000000: return 241; // -0.5
|
|
case 0x3F800000: return 242; // 1.0
|
|
case 0xBF800000: return 243; // -1.0
|
|
case 0x40000000: return 244; // 2.0
|
|
case 0xC0000000: return 245; // -2.0
|
|
case 0x40800000: return 246; // 4.0
|
|
case 0xC0800000: return 247; // -4.0
|
|
case 0x3E22F983: return 248; // 1.0 / (2.0 * pi)
|
|
default: break;
|
|
}
|
|
// clang-format on
|
|
}
|
|
|
|
return {};
|
|
}
|
|
|
|
// Encoding of the literal as an inline constant for a V_PK_*_IU16 instruction
|
|
// or nullopt.
|
|
std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {
|
|
return getInlineEncodingV216(false, Literal);
|
|
}
|
|
|
|
// Encoding of the literal as an inline constant for a V_PK_*_BF16 instruction
|
|
// or nullopt.
|
|
std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {
|
|
int32_t Signed = static_cast<int32_t>(Literal);
|
|
if (Signed >= 0 && Signed <= 64)
|
|
return 128 + Signed;
|
|
|
|
if (Signed >= -16 && Signed <= -1)
|
|
return 192 + std::abs(Signed);
|
|
|
|
// clang-format off
|
|
switch (Literal) {
|
|
case 0x3F00: return 240; // 0.5
|
|
case 0xBF00: return 241; // -0.5
|
|
case 0x3F80: return 242; // 1.0
|
|
case 0xBF80: return 243; // -1.0
|
|
case 0x4000: return 244; // 2.0
|
|
case 0xC000: return 245; // -2.0
|
|
case 0x4080: return 246; // 4.0
|
|
case 0xC080: return 247; // -4.0
|
|
case 0x3E22: return 248; // 1.0 / (2.0 * pi)
|
|
default: break;
|
|
}
|
|
// clang-format on
|
|
|
|
return std::nullopt;
|
|
}
|
|
|
|
// Encoding of the literal as an inline constant for a V_PK_*_F16 instruction
|
|
// or nullopt.
|
|
std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {
|
|
return getInlineEncodingV216(true, Literal);
|
|
}
|
|
|
|
// Whether the given literal can be inlined for a V_PK_* instruction.
|
|
bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType) {
|
|
switch (OpType) {
|
|
case AMDGPU::OPERAND_REG_IMM_V2INT16:
|
|
case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
|
|
return getInlineEncodingV216(false, Literal).has_value();
|
|
case AMDGPU::OPERAND_REG_IMM_V2FP16:
|
|
case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
|
|
return getInlineEncodingV216(true, Literal).has_value();
|
|
case AMDGPU::OPERAND_REG_IMM_V2BF16:
|
|
case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
|
|
return isInlinableLiteralV2BF16(Literal);
|
|
case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
|
|
return false;
|
|
default:
|
|
llvm_unreachable("bad packed operand type");
|
|
}
|
|
}
|
|
|
|
// Whether the given literal can be inlined for a V_PK_*_IU16 instruction.
|
|
bool isInlinableLiteralV2I16(uint32_t Literal) {
|
|
return getInlineEncodingV2I16(Literal).has_value();
|
|
}
|
|
|
|
// Whether the given literal can be inlined for a V_PK_*_BF16 instruction.
|
|
bool isInlinableLiteralV2BF16(uint32_t Literal) {
|
|
return getInlineEncodingV2BF16(Literal).has_value();
|
|
}
|
|
|
|
// Whether the given literal can be inlined for a V_PK_*_F16 instruction.
|
|
bool isInlinableLiteralV2F16(uint32_t Literal) {
|
|
return getInlineEncodingV2F16(Literal).has_value();
|
|
}
|
|
|
|
bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
|
|
if (IsFP64)
|
|
return !Lo_32(Val);
|
|
|
|
return isUInt<32>(Val) || isInt<32>(Val);
|
|
}
|
|
|
|
bool isArgPassedInSGPR(const Argument *A) {
|
|
const Function *F = A->getParent();
|
|
|
|
// Arguments to compute shaders are never a source of divergence.
|
|
CallingConv::ID CC = F->getCallingConv();
|
|
switch (CC) {
|
|
case CallingConv::AMDGPU_KERNEL:
|
|
case CallingConv::SPIR_KERNEL:
|
|
return true;
|
|
case CallingConv::AMDGPU_VS:
|
|
case CallingConv::AMDGPU_LS:
|
|
case CallingConv::AMDGPU_HS:
|
|
case CallingConv::AMDGPU_ES:
|
|
case CallingConv::AMDGPU_GS:
|
|
case CallingConv::AMDGPU_PS:
|
|
case CallingConv::AMDGPU_CS:
|
|
case CallingConv::AMDGPU_Gfx:
|
|
case CallingConv::AMDGPU_CS_Chain:
|
|
case CallingConv::AMDGPU_CS_ChainPreserve:
|
|
// For non-compute shaders, SGPR inputs are marked with either inreg or
|
|
// byval. Everything else is in VGPRs.
|
|
return A->hasAttribute(Attribute::InReg) ||
|
|
A->hasAttribute(Attribute::ByVal);
|
|
default:
|
|
// TODO: treat i1 as divergent?
|
|
return A->hasAttribute(Attribute::InReg);
|
|
}
|
|
}
|
|
|
|
bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo) {
|
|
// Arguments to compute shaders are never a source of divergence.
|
|
CallingConv::ID CC = CB->getCallingConv();
|
|
switch (CC) {
|
|
case CallingConv::AMDGPU_KERNEL:
|
|
case CallingConv::SPIR_KERNEL:
|
|
return true;
|
|
case CallingConv::AMDGPU_VS:
|
|
case CallingConv::AMDGPU_LS:
|
|
case CallingConv::AMDGPU_HS:
|
|
case CallingConv::AMDGPU_ES:
|
|
case CallingConv::AMDGPU_GS:
|
|
case CallingConv::AMDGPU_PS:
|
|
case CallingConv::AMDGPU_CS:
|
|
case CallingConv::AMDGPU_Gfx:
|
|
case CallingConv::AMDGPU_CS_Chain:
|
|
case CallingConv::AMDGPU_CS_ChainPreserve:
|
|
// For non-compute shaders, SGPR inputs are marked with either inreg or
|
|
// byval. Everything else is in VGPRs.
|
|
return CB->paramHasAttr(ArgNo, Attribute::InReg) ||
|
|
CB->paramHasAttr(ArgNo, Attribute::ByVal);
|
|
default:
|
|
return CB->paramHasAttr(ArgNo, Attribute::InReg);
|
|
}
|
|
}
|
|
|
|
static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
|
|
return isGCN3Encoding(ST) || isGFX10Plus(ST);
|
|
}
|
|
|
|
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
|
|
int64_t EncodedOffset) {
|
|
if (isGFX12Plus(ST))
|
|
return isUInt<23>(EncodedOffset);
|
|
|
|
return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
|
|
: isUInt<8>(EncodedOffset);
|
|
}
|
|
|
|
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
|
|
int64_t EncodedOffset, bool IsBuffer) {
|
|
if (isGFX12Plus(ST))
|
|
return isInt<24>(EncodedOffset);
|
|
|
|
return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<21>(EncodedOffset);
|
|
}
|
|
|
|
static bool isDwordAligned(uint64_t ByteOffset) {
|
|
return (ByteOffset & 3) == 0;
|
|
}
|
|
|
|
uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
|
|
uint64_t ByteOffset) {
|
|
if (hasSMEMByteOffset(ST))
|
|
return ByteOffset;
|
|
|
|
assert(isDwordAligned(ByteOffset));
|
|
return ByteOffset >> 2;
|
|
}
|
|
|
|
std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
|
|
int64_t ByteOffset, bool IsBuffer,
|
|
bool HasSOffset) {
|
|
// For unbuffered smem loads, it is illegal for the Immediate Offset to be
|
|
// negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
|
|
// Handle case where SOffset is not present.
|
|
if (!IsBuffer && !HasSOffset && ByteOffset < 0 && hasSMRDSignedImmOffset(ST))
|
|
return std::nullopt;
|
|
|
|
if (isGFX12Plus(ST)) // 24 bit signed offsets
|
|
return isInt<24>(ByteOffset) ? std::optional<int64_t>(ByteOffset)
|
|
: std::nullopt;
|
|
|
|
// The signed version is always a byte offset.
|
|
if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
|
|
assert(hasSMEMByteOffset(ST));
|
|
return isInt<20>(ByteOffset) ? std::optional<int64_t>(ByteOffset)
|
|
: std::nullopt;
|
|
}
|
|
|
|
if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
|
|
return std::nullopt;
|
|
|
|
int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
|
|
return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
|
|
? std::optional<int64_t>(EncodedOffset)
|
|
: std::nullopt;
|
|
}
|
|
|
|
std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
|
|
int64_t ByteOffset) {
|
|
if (!isCI(ST) || !isDwordAligned(ByteOffset))
|
|
return std::nullopt;
|
|
|
|
int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
|
|
return isUInt<32>(EncodedOffset) ? std::optional<int64_t>(EncodedOffset)
|
|
: std::nullopt;
|
|
}
|
|
|
|
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {
|
|
if (AMDGPU::isGFX10(ST))
|
|
return 12;
|
|
|
|
if (AMDGPU::isGFX12(ST))
|
|
return 24;
|
|
return 13;
|
|
}
|
|
|
|
namespace {
|
|
|
|
struct SourceOfDivergence {
|
|
unsigned Intr;
|
|
};
|
|
const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
|
|
|
|
struct AlwaysUniform {
|
|
unsigned Intr;
|
|
};
|
|
const AlwaysUniform *lookupAlwaysUniform(unsigned Intr);
|
|
|
|
#define GET_SourcesOfDivergence_IMPL
|
|
#define GET_UniformIntrinsics_IMPL
|
|
#define GET_Gfx9BufferFormat_IMPL
|
|
#define GET_Gfx10BufferFormat_IMPL
|
|
#define GET_Gfx11PlusBufferFormat_IMPL
|
|
|
|
#include "AMDGPUGenSearchableTables.inc"
|
|
|
|
} // end anonymous namespace
|
|
|
|
bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
|
|
return lookupSourceOfDivergence(IntrID);
|
|
}
|
|
|
|
bool isIntrinsicAlwaysUniform(unsigned IntrID) {
|
|
return lookupAlwaysUniform(IntrID);
|
|
}
|
|
|
|
const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
|
|
uint8_t NumComponents,
|
|
uint8_t NumFormat,
|
|
const MCSubtargetInfo &STI) {
|
|
return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(
|
|
BitsPerComp, NumComponents, NumFormat)
|
|
: isGFX10(STI)
|
|
? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)
|
|
: getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
|
|
}
|
|
|
|
const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
|
|
const MCSubtargetInfo &STI) {
|
|
return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)
|
|
: isGFX10(STI) ? getGfx10BufferFormatInfo(Format)
|
|
: getGfx9BufferFormatInfo(Format);
|
|
}
|
|
|
|
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
|
|
uint64_t TSFlags = MII.get(Opcode).TSFlags;
|
|
|
|
if (TSFlags & SIInstrFlags::SMRD)
|
|
return !getSMEMIsBuffer(Opcode);
|
|
if (!(TSFlags & SIInstrFlags::FLAT))
|
|
return false;
|
|
|
|
// Only SV and SVS modes are supported.
|
|
if (TSFlags & SIInstrFlags::FlatScratch)
|
|
return hasNamedOperand(Opcode, OpName::vaddr);
|
|
|
|
// Only GVS mode is supported.
|
|
return hasNamedOperand(Opcode, OpName::vaddr) &&
|
|
hasNamedOperand(Opcode, OpName::saddr);
|
|
|
|
return false;
|
|
}
|
|
|
|
bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc) {
|
|
for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {
|
|
int Idx = getNamedOperandIdx(OpDesc.getOpcode(), OpName);
|
|
if (Idx == -1)
|
|
continue;
|
|
|
|
if (OpDesc.operands()[Idx].RegClass == AMDGPU::VReg_64RegClassID ||
|
|
OpDesc.operands()[Idx].RegClass == AMDGPU::VReg_64_Align2RegClassID)
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool isDPALU_DPP32BitOpc(unsigned Opc) {
|
|
switch (Opc) {
|
|
case AMDGPU::V_MUL_LO_U32_e64:
|
|
case AMDGPU::V_MUL_LO_U32_e64_dpp:
|
|
case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
|
|
case AMDGPU::V_MUL_HI_U32_e64:
|
|
case AMDGPU::V_MUL_HI_U32_e64_dpp:
|
|
case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
|
|
case AMDGPU::V_MUL_HI_I32_e64:
|
|
case AMDGPU::V_MUL_HI_I32_e64_dpp:
|
|
case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
|
|
case AMDGPU::V_MAD_U32_e64:
|
|
case AMDGPU::V_MAD_U32_e64_dpp:
|
|
case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCSubtargetInfo &ST) {
|
|
if (!ST.hasFeature(AMDGPU::FeatureDPALU_DPP))
|
|
return false;
|
|
|
|
if (isDPALU_DPP32BitOpc(OpDesc.getOpcode()))
|
|
return ST.hasFeature(AMDGPU::FeatureGFX1250Insts);
|
|
|
|
return hasAny64BitVGPROperands(OpDesc);
|
|
}
|
|
|
|
unsigned getLdsDwGranularity(const MCSubtargetInfo &ST) {
|
|
return ST.hasFeature(AMDGPU::FeatureAddressableLocalMemorySize327680) ? 256
|
|
: 128;
|
|
}
|
|
|
|
bool isPackedFP32Inst(unsigned Opc) {
|
|
switch (Opc) {
|
|
case AMDGPU::V_PK_ADD_F32:
|
|
case AMDGPU::V_PK_ADD_F32_gfx12:
|
|
case AMDGPU::V_PK_MUL_F32:
|
|
case AMDGPU::V_PK_MUL_F32_gfx12:
|
|
case AMDGPU::V_PK_FMA_F32:
|
|
case AMDGPU::V_PK_FMA_F32_gfx12:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
} // namespace AMDGPU
|
|
|
|
raw_ostream &operator<<(raw_ostream &OS,
|
|
const AMDGPU::IsaInfo::TargetIDSetting S) {
|
|
switch (S) {
|
|
case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported):
|
|
OS << "Unsupported";
|
|
break;
|
|
case (AMDGPU::IsaInfo::TargetIDSetting::Any):
|
|
OS << "Any";
|
|
break;
|
|
case (AMDGPU::IsaInfo::TargetIDSetting::Off):
|
|
OS << "Off";
|
|
break;
|
|
case (AMDGPU::IsaInfo::TargetIDSetting::On):
|
|
OS << "On";
|
|
break;
|
|
}
|
|
return OS;
|
|
}
|
|
|
|
} // namespace llvm
|