shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[AMDGPU] Restrict packed math FP32 instructions to read only one SGPR per operand on gfx12+ (#152465)

Browse Source 
Sec. 4.6.7.1 of the gfx1250 SPG states that if an SGPR is used
as an operand, only one SGPR will be read for both the low and high
operations. As a result, the corresponding bits in `op_sel` and
`op_sel_hi` must be the same when the operand is an SGPR.

Co-authored-by: Tian, Shilei <Shilei.Tian@amd.com>

Co-authored-by: Tian, Shilei <Shilei.Tian@amd.com>
This commit is contained in:
Stanislav Mekhanoshin 2025-08-07 16:13:34 -07:00 committed by GitHub
parent cb2d56ce96
commit dddeb07c2e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 754 additions and 372 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
View File

@ -4933,6 +4933,43 @@ bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
return false;
}
// Packed math FP32 instructions typically accept SGPRs or VGPRs as source
// operands. On gfx12+, if a source operand uses SGPRs, the HW can only read
// the first SGPR and use it for both the low and high operations.
if (isPackedFP32Inst(Opc) && isGFX12Plus()) {
int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
const MCOperand &Src0 = Inst.getOperand(Src0Idx);
const MCOperand &Src1 = Inst.getOperand(Src1Idx);
unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
unsigned OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
const MCRegisterInfo *TRI = getContext().getRegisterInfo();
auto VerifyOneSGPR = [OpSel, OpSelHi](unsigned Index) -> bool {
unsigned Mask = 1U << Index;
return ((OpSel & Mask) == 0) && ((OpSelHi & Mask) == 0);
};
if (Src0.isReg() && isSGPR(Src0.getReg(), TRI) &&
!VerifyOneSGPR(/*Index=*/0))
return false;
if (Src1.isReg() && isSGPR(Src1.getReg(), TRI) &&
!VerifyOneSGPR(/*Index=*/1))
return false;
int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
if (Src2Idx != -1) {
const MCOperand &Src2 = Inst.getOperand(Src2Idx);
if (Src2.isReg() && isSGPR(Src2.getReg(), TRI) &&
!VerifyOneSGPR(/*Index=*/2))
return false;
}
}
return true;
}

67
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
View File

@ -18,6 +18,7 @@
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/LiveIntervals.h"
@ -5534,6 +5535,15 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
}
// See SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more
// information.
if (AMDGPU::isPackedFP32Inst(Opcode) && AMDGPU::isGFX12Plus(ST)) {
for (unsigned I = 0; I < 3; ++I) {
if (!isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, I))
return false;
}
}
return true;
}
@ -6005,6 +6015,21 @@ bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];
unsigned Opc = MI.getOpcode();
// See SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more
// information.
if (AMDGPU::isPackedFP32Inst(MI.getOpcode()) && AMDGPU::isGFX12Plus(ST) &&
MO.isReg() && RI.isSGPRReg(MRI, MO.getReg())) {
constexpr const AMDGPU::OpName OpNames[] = {
AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
for (auto [I, OpName] : enumerate(OpNames)) {
int SrcIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[I]);
if (static_cast<unsigned>(SrcIdx) == OpIdx &&
!isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, I, &MO))
return false;
}
}
if (!isLegalRegOperand(MRI, OpInfo, MO))
return false;
@ -6053,6 +6078,39 @@ bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
return true;
}
bool SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand(
const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
const MachineOperand *MO) const {
constexpr const unsigned NumOps = 3;
constexpr const AMDGPU::OpName OpNames[NumOps * 2] = {
AMDGPU::OpName::src0, AMDGPU::OpName::src1,
AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,
AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
assert(SrcN < NumOps);
if (!MO) {
int SrcIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[SrcN]);
if (SrcIdx == -1)
return true;
MO = &MI.getOperand(SrcIdx);
}
if (!MO->isReg() || !RI.isSGPRReg(MRI, MO->getReg()))
return true;
int ModsIdx =
AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[NumOps + SrcN]);
if (ModsIdx == -1)
return true;
unsigned Mods = MI.getOperand(ModsIdx).getImm();
bool OpSel = Mods & SISrcMods::OP_SEL_0;
bool OpSelHi = Mods & SISrcMods::OP_SEL_1;
return !OpSel && !OpSelHi;
}
bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
const MachineOperand *MO) const {
const MachineFunction &MF = *MI.getParent()->getParent();
@ -6390,6 +6448,15 @@ void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
!RI.isVGPR(MRI, MI.getOperand(VOP3Idx[2]).getReg()))
legalizeOpWithMove(MI, VOP3Idx[2]);
// Fix the register class of packed FP32 instructions on gfx12+. See
// SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more information.
if (AMDGPU::isPackedFP32Inst(Opc) && AMDGPU::isGFX12Plus(ST)) {
for (unsigned I = 0; I < 3; ++I) {
if (!isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, /*SrcN=*/I))
legalizeOpWithMove(MI, VOP3Idx[I]);
}
}
}
Register SIInstrInfo::readlaneVGPRToSGPR(

13
llvm/lib/Target/AMDGPU/SIInstrInfo.h
View File

@ -1287,6 +1287,19 @@ public:
const MachineOperand &MO) const;
bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
const MachineOperand &MO) const;
/// Check if \p MO would be a legal operand for gfx12+ packed math FP32
/// instructions. Packed math FP32 instructions typically accept SGPRs or
/// VGPRs as source operands. On gfx12+, if a source operand uses SGPRs, the
/// HW can only read the first SGPR and use it for both the low and high
/// operations.
/// \p SrcN can be 0, 1, or 2, representing src0, src1, and src2,
/// respectively. If \p MO is nullptr, the operand corresponding to SrcN will
/// be used.
bool isLegalGFX12PlusPackedMathFP32Operand(
const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
const MachineOperand *MO = nullptr) const;
/// Legalize operands in \p MI by either commuting it or inserting a
/// copy of src1.
void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;

14
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
View File

@ -3318,6 +3318,20 @@ unsigned getLdsDwGranularity(const MCSubtargetInfo &ST) {
return 128;
}
bool isPackedFP32Inst(unsigned Opc) {
switch (Opc) {
case AMDGPU::V_PK_ADD_F32:
case AMDGPU::V_PK_ADD_F32_gfx12:
case AMDGPU::V_PK_MUL_F32:
case AMDGPU::V_PK_MUL_F32_gfx12:
case AMDGPU::V_PK_FMA_F32:
case AMDGPU::V_PK_FMA_F32_gfx12:
return true;
default:
return false;
}
}
} // namespace AMDGPU
raw_ostream &operator<<(raw_ostream &OS,

2
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
View File

@ -1709,6 +1709,8 @@ bool isArgPassedInSGPR(const Argument *Arg);
bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo);
LLVM_READONLY bool isPackedFP32Inst(unsigned Opc);
LLVM_READONLY
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
int64_t EncodedOffset);

919
llvm/test/CodeGen/AMDGPU/packed-fp32.ll
View File

File diff suppressed because it is too large Load Diff

74
llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_err.s Normal file
View File

@ -0,0 +1,74 @@
// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
v_pk_fma_f32 v[8:9], s[0:1], v[0:1], v[4:5]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], v[0:1], s[0:1], v[4:5]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], v[0:1], v[4:5], s[0:1]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], s[0:1], v[0:1], v[4:5] op_sel:[1,0,0] op_sel_hi:[0,0,0]
// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], s[0:1], v[0:1], v[4:5] op_sel:[1,0,0] op_sel_hi:[1,0,0]
// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], v[0:1], s[0:1], v[4:5] op_sel:[0,1,0] op_sel_hi:[0,0,0]
// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], v[0:1], v[4:5], s[0:1] op_sel:[0,0,1] op_sel_hi:[0,0,0]
// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
v_pk_mul_f32 v[8:9], s[0:1], v[0:1]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_mul_f32 v[8:9], v[0:1], s[0:1]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_mul_f32 v[8:9], s[0:1], v[0:1] op_sel:[1,0] op_sel_hi:[0,0]
// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
v_pk_mul_f32 v[8:9], v[0:1], s[0:1] op_sel:[0,1] op_sel_hi:[0,0]
// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
v_pk_mul_f32 v[8:9], v[0:1], s[0:1] op_sel:[0,1] op_sel_hi:[0,1]
// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
v_pk_add_f32 v[8:9], s[0:1], v[0:1]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_add_f32 v[8:9], v[0:1], s[0:1]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_add_f32 v[8:9], s[0:1], v[0:1] op_sel:[1,0] op_sel_hi:[0,0]
// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
v_pk_add_f32 v[8:9], v[0:1], s[0:1] op_sel:[0,1] op_sel_hi:[0,0]
// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
v_pk_add_f32 v[8:9], v[0:1], s[0:1] op_sel:[0,1] op_sel_hi:[0,1]
// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], exec, v[0:1], v[4:5]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], v[0:1], exec, v[4:5]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_fma_f32 v[8:9], v[0:1], v[4:5], exec
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_mul_f32 v[8:9], exec, v[0:1]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_mul_f32 v[8:9], v[0:1], exec
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_add_f32 v[8:9], exec, v[0:1]
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
v_pk_add_f32 v[8:9], v[0:1], exec
// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand