shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
Vyacheslav Levytskyy ebdadcfeb9
[SPIR-V] Improve correctness of emitted MIR between passes for branching instructions (#106966) 
This PR improves correctness of emitted MIR between passes for branching
instructions and thus increase number of passing tests when expensive
checks are on. Specifically, we address here such issues with machine
verifier as:
* fix switch generation: generate correct successors and undo the
"address taken" status to reflect that a successor doesn't actually
correspond to an IR-level basic block;
* fix incorrect definition of OpBranch and OpBranchConditional in
TableGen (SPIRVInstrInfo.td) to set isBarrier status properly and set a
correct type of virtual registers;
* fix a case when Phi refers to a type definition that goes after the
Phi instruction, so that the virtual register definition of the type
doesn't dominate all uses.

This PR decrease number of failing tests under expensive checks from 56
to 50.
2024-09-03 19:02:03 +02:00

911 lines
36 KiB
C++
Raw Blame History

//===-- SPIRVPreLegalizer.cpp - prepare IR for legalization -----*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// The pass prepares IR for legalization: it assigns SPIR-V types to registers
// and removes intrinsics which holded these types during IR translation.
// Also it processes constants and registers them in GR to avoid duplication.
//
//===----------------------------------------------------------------------===//
#include "SPIRV.h"
#include "SPIRVSubtarget.h"
#include "SPIRVUtils.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#include "llvm/Target/TargetIntrinsicInfo.h"
#define DEBUG_TYPE "spirv-prelegalizer"
using namespace llvm;
namespace {
class SPIRVPreLegalizer : public MachineFunctionPass {
public:
static char ID;
SPIRVPreLegalizer() : MachineFunctionPass(ID) {
initializeSPIRVPreLegalizerPass(*PassRegistry::getPassRegistry());
}
bool runOnMachineFunction(MachineFunction &MF) override;
};
} // namespace
static void
addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
const SPIRVSubtarget &STI,
DenseMap<MachineInstr *, Type *> &TargetExtConstTypes,
SmallSet<Register, 4> &TrackedConstRegs) {
MachineRegisterInfo &MRI = MF.getRegInfo();
DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
SmallVector<MachineInstr *, 10> ToErase, ToEraseComposites;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (!isSpvIntrinsic(MI, Intrinsic::spv_track_constant))
continue;
ToErase.push_back(&MI);
Register SrcReg = MI.getOperand(2).getReg();
auto *Const =
cast<Constant>(cast<ConstantAsMetadata>(
MI.getOperand(3).getMetadata()->getOperand(0))
->getValue());
if (auto *GV = dyn_cast<GlobalValue>(Const)) {
Register Reg = GR->find(GV, &MF);
if (!Reg.isValid())
GR->add(GV, &MF, SrcReg);
else
RegsAlreadyAddedToDT[&MI] = Reg;
} else {
Register Reg = GR->find(Const, &MF);
if (!Reg.isValid()) {
if (auto *ConstVec = dyn_cast<ConstantDataVector>(Const)) {
auto *BuildVec = MRI.getVRegDef(SrcReg);
assert(BuildVec &&
BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR);
for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) {
// Ensure that OpConstantComposite reuses a constant when it's
// already created and available in the same machine function.
Constant *ElemConst = ConstVec->getElementAsConstant(i);
Register ElemReg = GR->find(ElemConst, &MF);
if (!ElemReg.isValid())
GR->add(ElemConst, &MF, BuildVec->getOperand(1 + i).getReg());
else
BuildVec->getOperand(1 + i).setReg(ElemReg);
}
}
GR->add(Const, &MF, SrcReg);
TrackedConstRegs.insert(SrcReg);
if (Const->getType()->isTargetExtTy()) {
// remember association so that we can restore it when assign types
MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
if (SrcMI && (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT ||
SrcMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF))
TargetExtConstTypes[SrcMI] = Const->getType();
if (Const->isNullValue()) {
MachineIRBuilder MIB(MF);
SPIRVType *ExtType =
GR->getOrCreateSPIRVType(Const->getType(), MIB);
SrcMI->setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull));
SrcMI->addOperand(MachineOperand::CreateReg(
GR->getSPIRVTypeID(ExtType), false));
}
}
} else {
RegsAlreadyAddedToDT[&MI] = Reg;
// This MI is unused and will be removed. If the MI uses
// const_composite, it will be unused and should be removed too.
assert(MI.getOperand(2).isReg() && "Reg operand is expected");
MachineInstr *SrcMI = MRI.getVRegDef(MI.getOperand(2).getReg());
if (SrcMI && isSpvIntrinsic(*SrcMI, Intrinsic::spv_const_composite))
ToEraseComposites.push_back(SrcMI);
}
}
}
}
for (MachineInstr *MI : ToErase) {
Register Reg = MI->getOperand(2).getReg();
if (RegsAlreadyAddedToDT.contains(MI))
Reg = RegsAlreadyAddedToDT[MI];
auto *RC = MRI.getRegClassOrNull(MI->getOperand(0).getReg());
if (!MRI.getRegClassOrNull(Reg) && RC)
MRI.setRegClass(Reg, RC);
MRI.replaceRegWith(MI->getOperand(0).getReg(), Reg);
MI->eraseFromParent();
}
for (MachineInstr *MI : ToEraseComposites)
MI->eraseFromParent();
}
static void
foldConstantsIntoIntrinsics(MachineFunction &MF,
const SmallSet<Register, 4> &TrackedConstRegs) {
SmallVector<MachineInstr *, 10> ToErase;
MachineRegisterInfo &MRI = MF.getRegInfo();
const unsigned AssignNameOperandShift = 2;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (!isSpvIntrinsic(MI, Intrinsic::spv_assign_name))
continue;
unsigned NumOp = MI.getNumExplicitDefs() + AssignNameOperandShift;
while (MI.getOperand(NumOp).isReg()) {
MachineOperand &MOp = MI.getOperand(NumOp);
MachineInstr *ConstMI = MRI.getVRegDef(MOp.getReg());
assert(ConstMI->getOpcode() == TargetOpcode::G_CONSTANT);
MI.removeOperand(NumOp);
MI.addOperand(MachineOperand::CreateImm(
ConstMI->getOperand(1).getCImm()->getZExtValue()));
Register DefReg = ConstMI->getOperand(0).getReg();
if (MRI.use_empty(DefReg) && !TrackedConstRegs.contains(DefReg))
ToErase.push_back(ConstMI);
}
}
}
for (MachineInstr *MI : ToErase)
MI->eraseFromParent();
}
static MachineInstr *findAssignTypeInstr(Register Reg,
MachineRegisterInfo *MRI) {
for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(Reg),
IE = MRI->use_instr_end();
I != IE; ++I) {
MachineInstr *UseMI = &*I;
if ((isSpvIntrinsic(*UseMI, Intrinsic::spv_assign_ptr_type) ||
isSpvIntrinsic(*UseMI, Intrinsic::spv_assign_type)) &&
UseMI->getOperand(1).getReg() == Reg)
return UseMI;
}
return nullptr;
}
static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,
MachineIRBuilder MIB) {
// Get access to information about available extensions
const SPIRVSubtarget *ST =
static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
SmallVector<MachineInstr *, 10> ToErase;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (!isSpvIntrinsic(MI, Intrinsic::spv_bitcast) &&
!isSpvIntrinsic(MI, Intrinsic::spv_ptrcast))
continue;
assert(MI.getOperand(2).isReg());
MIB.setInsertPt(*MI.getParent(), MI);
ToErase.push_back(&MI);
if (isSpvIntrinsic(MI, Intrinsic::spv_bitcast)) {
MIB.buildBitcast(MI.getOperand(0).getReg(), MI.getOperand(2).getReg());
continue;
}
Register Def = MI.getOperand(0).getReg();
Register Source = MI.getOperand(2).getReg();
Type *ElemTy = getMDOperandAsType(MI.getOperand(3).getMetadata(), 0);
SPIRVType *BaseTy = GR->getOrCreateSPIRVType(ElemTy, MIB);
SPIRVType *AssignedPtrType = GR->getOrCreateSPIRVPointerType(
BaseTy, MI, *MF.getSubtarget<SPIRVSubtarget>().getInstrInfo(),
addressSpaceToStorageClass(MI.getOperand(4).getImm(), *ST));
// If the ptrcast would be redundant, replace all uses with the source
// register.
MachineRegisterInfo *MRI = MIB.getMRI();
if (GR->getSPIRVTypeForVReg(Source) == AssignedPtrType) {
// Erase Def's assign type instruction if we are going to replace Def.
if (MachineInstr *AssignMI = findAssignTypeInstr(Def, MRI))
ToErase.push_back(AssignMI);
MRI->replaceRegWith(Def, Source);
} else {
GR->assignSPIRVTypeToVReg(AssignedPtrType, Def, MF);
MIB.buildBitcast(Def, Source);
// MachineVerifier requires that bitcast must change the type.
// Change AddressSpace if needed to hint that Def and Source points to
// different types: this doesn't change actual code generation.
LLT DefType = MRI->getType(Def);
if (DefType == MRI->getType(Source))
MRI->setType(Def,
LLT::pointer((DefType.getAddressSpace() + 1) %
SPIRVSubtarget::MaxLegalAddressSpace,
GR->getPointerSize()));
}
}
}
for (MachineInstr *MI : ToErase)
MI->eraseFromParent();
}
// Translating GV, IRTranslator sometimes generates following IR:
// %1 = G_GLOBAL_VALUE
// %2 = COPY %1
// %3 = G_ADDRSPACE_CAST %2
//
// or
//
// %1 = G_ZEXT %2
// G_MEMCPY ... %2 ...
//
// New registers have no SPIRVType and no register class info.
//
// Set SPIRVType for GV, propagate it from GV to other instructions,
// also set register classes.
static SPIRVType *propagateSPIRVType(MachineInstr *MI, SPIRVGlobalRegistry *GR,
MachineRegisterInfo &MRI,
MachineIRBuilder &MIB) {
SPIRVType *SpvType = nullptr;
assert(MI && "Machine instr is expected");
if (MI->getOperand(0).isReg()) {
Register Reg = MI->getOperand(0).getReg();
SpvType = GR->getSPIRVTypeForVReg(Reg);
if (!SpvType) {
switch (MI->getOpcode()) {
case TargetOpcode::G_CONSTANT: {
MIB.setInsertPt(*MI->getParent(), MI);
Type *Ty = MI->getOperand(1).getCImm()->getType();
SpvType = GR->getOrCreateSPIRVType(Ty, MIB);
break;
}
case TargetOpcode::G_GLOBAL_VALUE: {
MIB.setInsertPt(*MI->getParent(), MI);
const GlobalValue *Global = MI->getOperand(1).getGlobal();
Type *ElementTy = toTypedPointer(GR->getDeducedGlobalValueType(Global));
auto *Ty = TypedPointerType::get(ElementTy,
Global->getType()->getAddressSpace());
SpvType = GR->getOrCreateSPIRVType(Ty, MIB);
break;
}
case TargetOpcode::G_ANYEXT:
case TargetOpcode::G_SEXT:
case TargetOpcode::G_ZEXT: {
if (MI->getOperand(1).isReg()) {
if (MachineInstr *DefInstr =
MRI.getVRegDef(MI->getOperand(1).getReg())) {
if (SPIRVType *Def = propagateSPIRVType(DefInstr, GR, MRI, MIB)) {
unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Def);
unsigned ExpectedBW =
std::max(MRI.getType(Reg).getScalarSizeInBits(), CurrentBW);
unsigned NumElements = GR->getScalarOrVectorComponentCount(Def);
SpvType = GR->getOrCreateSPIRVIntegerType(ExpectedBW, MIB);
if (NumElements > 1)
SpvType =
GR->getOrCreateSPIRVVectorType(SpvType, NumElements, MIB);
}
}
}
break;
}
case TargetOpcode::G_PTRTOINT:
SpvType = GR->getOrCreateSPIRVIntegerType(
MRI.getType(Reg).getScalarSizeInBits(), MIB);
break;
case TargetOpcode::G_TRUNC:
case TargetOpcode::G_ADDRSPACE_CAST:
case TargetOpcode::G_PTR_ADD:
case TargetOpcode::COPY: {
MachineOperand &Op = MI->getOperand(1);
MachineInstr *Def = Op.isReg() ? MRI.getVRegDef(Op.getReg()) : nullptr;
if (Def)
SpvType = propagateSPIRVType(Def, GR, MRI, MIB);
break;
}
default:
break;
}
if (SpvType)
GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF());
if (!MRI.getRegClassOrNull(Reg))
MRI.setRegClass(Reg, SpvType ? GR->getRegClass(SpvType)
: &SPIRV::iIDRegClass);
}
}
return SpvType;
}
// To support current approach and limitations wrt. bit width here we widen a
// scalar register with a bit width greater than 1 to valid sizes and cap it to
// 64 width.
static void widenScalarLLTNextPow2(Register Reg, MachineRegisterInfo &MRI) {
LLT RegType = MRI.getType(Reg);
if (!RegType.isScalar())
return;
unsigned Sz = RegType.getScalarSizeInBits();
if (Sz == 1)
return;
unsigned NewSz = std::min(std::max(1u << Log2_32_Ceil(Sz), 8u), 64u);
if (NewSz != Sz)
MRI.setType(Reg, LLT::scalar(NewSz));
}
static std::pair<Register, unsigned>
createNewIdReg(SPIRVType *SpvType, Register SrcReg, MachineRegisterInfo &MRI,
const SPIRVGlobalRegistry &GR) {
if (!SpvType)
SpvType = GR.getSPIRVTypeForVReg(SrcReg);
const TargetRegisterClass *RC = GR.getRegClass(SpvType);
Register Reg = MRI.createGenericVirtualRegister(GR.getRegType(SpvType));
MRI.setRegClass(Reg, RC);
unsigned GetIdOp = SPIRV::GET_ID;
if (RC == &SPIRV::fIDRegClass)
GetIdOp = SPIRV::GET_fID;
else if (RC == &SPIRV::pIDRegClass)
GetIdOp = SPIRV::GET_pID;
else if (RC == &SPIRV::vfIDRegClass)
GetIdOp = SPIRV::GET_vfID;
else if (RC == &SPIRV::vpIDRegClass)
GetIdOp = SPIRV::GET_vpID;
else if (RC == &SPIRV::vIDRegClass)
GetIdOp = SPIRV::GET_vID;
return {Reg, GetIdOp};
}
// Insert ASSIGN_TYPE instuction between Reg and its definition, set NewReg as
// a dst of the definition, assign SPIRVType to both registers. If SpvType is
// provided, use it as SPIRVType in ASSIGN_TYPE, otherwise create it from Ty.
// It's used also in SPIRVBuiltins.cpp.
// TODO: maybe move to SPIRVUtils.
namespace llvm {
Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType,
SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) {
MachineInstr *Def = MRI.getVRegDef(Reg);
assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected.");
MIB.setInsertPt(*Def->getParent(),
(Def->getNextNode() ? Def->getNextNode()->getIterator()
: Def->getParent()->end()));
SpvType = SpvType ? SpvType : GR->getOrCreateSPIRVType(Ty, MIB);
Register NewReg = MRI.createGenericVirtualRegister(MRI.getType(Reg));
if (auto *RC = MRI.getRegClassOrNull(Reg)) {
MRI.setRegClass(NewReg, RC);
} else {
auto RegClass = GR->getRegClass(SpvType);
MRI.setRegClass(NewReg, RegClass);
MRI.setRegClass(Reg, RegClass);
}
GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF());
// This is to make it convenient for Legalizer to get the SPIRVType
// when processing the actual MI (i.e. not pseudo one).
GR->assignSPIRVTypeToVReg(SpvType, NewReg, MIB.getMF());
// Copy MIFlags from Def to ASSIGN_TYPE instruction. It's required to keep
// the flags after instruction selection.
const uint32_t Flags = Def->getFlags();
MIB.buildInstr(SPIRV::ASSIGN_TYPE)
.addDef(Reg)
.addUse(NewReg)
.addUse(GR->getSPIRVTypeID(SpvType))
.setMIFlags(Flags);
Def->getOperand(0).setReg(NewReg);
return NewReg;
}
void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR) {
assert(MI.getNumDefs() > 0 && MRI.hasOneUse(MI.getOperand(0).getReg()));
MachineInstr &AssignTypeInst =
*(MRI.use_instr_begin(MI.getOperand(0).getReg()));
auto NewReg =
createNewIdReg(nullptr, MI.getOperand(0).getReg(), MRI, *GR).first;
AssignTypeInst.getOperand(1).setReg(NewReg);
MI.getOperand(0).setReg(NewReg);
MIB.setInsertPt(*MI.getParent(),
(MI.getNextNode() ? MI.getNextNode()->getIterator()
: MI.getParent()->end()));
for (auto &Op : MI.operands()) {
if (!Op.isReg() || Op.isDef())
continue;
auto IdOpInfo = createNewIdReg(nullptr, Op.getReg(), MRI, *GR);
MIB.buildInstr(IdOpInfo.second).addDef(IdOpInfo.first).addUse(Op.getReg());
Op.setReg(IdOpInfo.first);
}
}
} // namespace llvm
static void
generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
MachineIRBuilder MIB,
DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {
// Get access to information about available extensions
const SPIRVSubtarget *ST =
static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());
MachineRegisterInfo &MRI = MF.getRegInfo();
SmallVector<MachineInstr *, 10> ToErase;
DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;
bool IsExtendedInts =
ST->canUseExtension(
SPIRV::Extension::SPV_INTEL_arbitrary_precision_integers) ||
ST->canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions);
for (MachineBasicBlock *MBB : post_order(&MF)) {
if (MBB->empty())
continue;
bool ReachedBegin = false;
for (auto MII = std::prev(MBB->end()), Begin = MBB->begin();
!ReachedBegin;) {
MachineInstr &MI = *MII;
unsigned MIOp = MI.getOpcode();
if (!IsExtendedInts) {
// validate bit width of scalar registers
for (const auto &MOP : MI.operands())
if (MOP.isReg())
widenScalarLLTNextPow2(MOP.getReg(), MRI);
}
if (isSpvIntrinsic(MI, Intrinsic::spv_assign_ptr_type)) {
Register Reg = MI.getOperand(1).getReg();
MIB.setInsertPt(*MI.getParent(), MI.getIterator());
Type *ElementTy = getMDOperandAsType(MI.getOperand(2).getMetadata(), 0);
SPIRVType *BaseTy = GR->getOrCreateSPIRVType(ElementTy, MIB);
SPIRVType *AssignedPtrType = GR->getOrCreateSPIRVPointerType(
BaseTy, MI, *MF.getSubtarget<SPIRVSubtarget>().getInstrInfo(),
addressSpaceToStorageClass(MI.getOperand(3).getImm(), *ST));
MachineInstr *Def = MRI.getVRegDef(Reg);
assert(Def && "Expecting an instruction that defines the register");
// G_GLOBAL_VALUE already has type info.
if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE &&
Def->getOpcode() != SPIRV::ASSIGN_TYPE)
insertAssignInstr(Reg, nullptr, AssignedPtrType, GR, MIB,
MF.getRegInfo());
ToErase.push_back(&MI);
} else if (isSpvIntrinsic(MI, Intrinsic::spv_assign_type)) {
Register Reg = MI.getOperand(1).getReg();
Type *Ty = getMDOperandAsType(MI.getOperand(2).getMetadata(), 0);
MachineInstr *Def = MRI.getVRegDef(Reg);
assert(Def && "Expecting an instruction that defines the register");
// G_GLOBAL_VALUE already has type info.
if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE &&
Def->getOpcode() != SPIRV::ASSIGN_TYPE)
insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo());
ToErase.push_back(&MI);
} else if (MIOp == TargetOpcode::G_CONSTANT ||
MIOp == TargetOpcode::G_FCONSTANT ||
MIOp == TargetOpcode::G_BUILD_VECTOR) {
// %rc = G_CONSTANT ty Val
// ===>
// %cty = OpType* ty
// %rctmp = G_CONSTANT ty Val
// %rc = ASSIGN_TYPE %rctmp, %cty
Register Reg = MI.getOperand(0).getReg();
bool NeedAssignType = true;
if (MRI.hasOneUse(Reg)) {
MachineInstr &UseMI = *MRI.use_instr_begin(Reg);
if (isSpvIntrinsic(UseMI, Intrinsic::spv_assign_type) ||
isSpvIntrinsic(UseMI, Intrinsic::spv_assign_name))
continue;
if (UseMI.getOpcode() == SPIRV::ASSIGN_TYPE)
NeedAssignType = false;
}
Type *Ty = nullptr;
if (MIOp == TargetOpcode::G_CONSTANT) {
auto TargetExtIt = TargetExtConstTypes.find(&MI);
Ty = TargetExtIt == TargetExtConstTypes.end()
? MI.getOperand(1).getCImm()->getType()
: TargetExtIt->second;
const ConstantInt *OpCI = MI.getOperand(1).getCImm();
Register PrimaryReg = GR->find(OpCI, &MF);
if (!PrimaryReg.isValid()) {
GR->add(OpCI, &MF, Reg);
} else if (PrimaryReg != Reg &&
MRI.getType(Reg) == MRI.getType(PrimaryReg)) {
auto *RCReg = MRI.getRegClassOrNull(Reg);
auto *RCPrimary = MRI.getRegClassOrNull(PrimaryReg);
if (!RCReg || RCPrimary == RCReg) {
RegsAlreadyAddedToDT[&MI] = PrimaryReg;
ToErase.push_back(&MI);
NeedAssignType = false;
}
}
} else if (MIOp == TargetOpcode::G_FCONSTANT) {
Ty = MI.getOperand(1).getFPImm()->getType();
} else {
assert(MIOp == TargetOpcode::G_BUILD_VECTOR);
Type *ElemTy = nullptr;
MachineInstr *ElemMI = MRI.getVRegDef(MI.getOperand(1).getReg());
assert(ElemMI);
if (ElemMI->getOpcode() == TargetOpcode::G_CONSTANT) {
ElemTy = ElemMI->getOperand(1).getCImm()->getType();
} else if (ElemMI->getOpcode() == TargetOpcode::G_FCONSTANT) {
ElemTy = ElemMI->getOperand(1).getFPImm()->getType();
} else {
// There may be a case when we already know Reg's type.
MachineInstr *NextMI = MI.getNextNode();
if (!NextMI || NextMI->getOpcode() != SPIRV::ASSIGN_TYPE ||
NextMI->getOperand(1).getReg() != Reg)
llvm_unreachable("Unexpected opcode");
}
if (ElemTy)
Ty = VectorType::get(
ElemTy, MI.getNumExplicitOperands() - MI.getNumExplicitDefs(),
false);
else
NeedAssignType = false;
}
if (NeedAssignType)
insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MRI);
} else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) {
propagateSPIRVType(&MI, GR, MRI, MIB);
}
if (MII == Begin)
ReachedBegin = true;
else
--MII;
}
}
for (MachineInstr *MI : ToErase) {
auto It = RegsAlreadyAddedToDT.find(MI);
if (RegsAlreadyAddedToDT.contains(MI))
MRI.replaceRegWith(MI->getOperand(0).getReg(), It->second);
MI->eraseFromParent();
}
// Address the case when IRTranslator introduces instructions with new
// registers without SPIRVType associated.
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
switch (MI.getOpcode()) {
case TargetOpcode::G_TRUNC:
case TargetOpcode::G_ANYEXT:
case TargetOpcode::G_SEXT:
case TargetOpcode::G_ZEXT:
case TargetOpcode::G_PTRTOINT:
case TargetOpcode::COPY:
case TargetOpcode::G_ADDRSPACE_CAST:
propagateSPIRVType(&MI, GR, MRI, MIB);
break;
}
}
}
}
// Defined in SPIRVLegalizerInfo.cpp.
extern bool isTypeFoldingSupported(unsigned Opcode);
static void processInstrsWithTypeFolding(MachineFunction &MF,
SPIRVGlobalRegistry *GR,
MachineIRBuilder MIB) {
MachineRegisterInfo &MRI = MF.getRegInfo();
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (isTypeFoldingSupported(MI.getOpcode()))
processInstr(MI, MIB, MRI, GR);
}
}
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
// We need to rewrite dst types for ASSIGN_TYPE instrs to be able
// to perform tblgen'erated selection and we can't do that on Legalizer
// as it operates on gMIR only.
if (MI.getOpcode() != SPIRV::ASSIGN_TYPE)
continue;
Register SrcReg = MI.getOperand(1).getReg();
unsigned Opcode = MRI.getVRegDef(SrcReg)->getOpcode();
if (!isTypeFoldingSupported(Opcode))
continue;
Register DstReg = MI.getOperand(0).getReg();
// Don't need to reset type of register holding constant and used in
// G_ADDRSPACE_CAST, since it breaks legalizer.
if (Opcode == TargetOpcode::G_CONSTANT && MRI.hasOneUse(DstReg)) {
MachineInstr &UseMI = *MRI.use_instr_begin(DstReg);
if (UseMI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST)
continue;
}
}
}
}
static Register
collectInlineAsmInstrOperands(MachineInstr *MI,
SmallVector<unsigned, 4> *Ops = nullptr) {
Register DefReg;
unsigned StartOp = InlineAsm::MIOp_FirstOperand,
AsmDescOp = InlineAsm::MIOp_FirstOperand;
for (unsigned Idx = StartOp, MISz = MI->getNumOperands(); Idx != MISz;
++Idx) {
const MachineOperand &MO = MI->getOperand(Idx);
if (MO.isMetadata())
continue;
if (Idx == AsmDescOp && MO.isImm()) {
// compute the index of the next operand descriptor
const InlineAsm::Flag F(MO.getImm());
AsmDescOp += 1 + F.getNumOperandRegisters();
continue;
}
if (MO.isReg() && MO.isDef()) {
if (!Ops)
return MO.getReg();
else
DefReg = MO.getReg();
} else if (Ops) {
Ops->push_back(Idx);
}
}
return DefReg;
}
static void
insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,
const SPIRVSubtarget &ST, MachineIRBuilder MIRBuilder,
const SmallVector<MachineInstr *> &ToProcess) {
MachineRegisterInfo &MRI = MF.getRegInfo();
Register AsmTargetReg;
for (unsigned i = 0, Sz = ToProcess.size(); i + 1 < Sz; i += 2) {
MachineInstr *I1 = ToProcess[i], *I2 = ToProcess[i + 1];
assert(isSpvIntrinsic(*I1, Intrinsic::spv_inline_asm) && I2->isInlineAsm());
MIRBuilder.setInsertPt(*I2->getParent(), *I2);
if (!AsmTargetReg.isValid()) {
// define vendor specific assembly target or dialect
AsmTargetReg = MRI.createGenericVirtualRegister(LLT::scalar(32));
MRI.setRegClass(AsmTargetReg, &SPIRV::iIDRegClass);
auto AsmTargetMIB =
MIRBuilder.buildInstr(SPIRV::OpAsmTargetINTEL).addDef(AsmTargetReg);
addStringImm(ST.getTargetTripleAsStr(), AsmTargetMIB);
GR->add(AsmTargetMIB.getInstr(), &MF, AsmTargetReg);
}
// create types
const MDNode *IAMD = I1->getOperand(1).getMetadata();
FunctionType *FTy = cast<FunctionType>(getMDOperandAsType(IAMD, 0));
SmallVector<SPIRVType *, 4> ArgTypes;
for (const auto &ArgTy : FTy->params())
ArgTypes.push_back(GR->getOrCreateSPIRVType(ArgTy, MIRBuilder));
SPIRVType *RetType =
GR->getOrCreateSPIRVType(FTy->getReturnType(), MIRBuilder);
SPIRVType *FuncType = GR->getOrCreateOpTypeFunctionWithArgs(
FTy, RetType, ArgTypes, MIRBuilder);
// define vendor specific assembly instructions string
Register AsmReg = MRI.createGenericVirtualRegister(LLT::scalar(32));
MRI.setRegClass(AsmReg, &SPIRV::iIDRegClass);
auto AsmMIB = MIRBuilder.buildInstr(SPIRV::OpAsmINTEL)
.addDef(AsmReg)
.addUse(GR->getSPIRVTypeID(RetType))
.addUse(GR->getSPIRVTypeID(FuncType))
.addUse(AsmTargetReg);
// inline asm string:
addStringImm(I2->getOperand(InlineAsm::MIOp_AsmString).getSymbolName(),
AsmMIB);
// inline asm constraint string:
addStringImm(cast<MDString>(I1->getOperand(2).getMetadata()->getOperand(0))
->getString(),
AsmMIB);
GR->add(AsmMIB.getInstr(), &MF, AsmReg);
// calls the inline assembly instruction
unsigned ExtraInfo = I2->getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
if (ExtraInfo & InlineAsm::Extra_HasSideEffects)
MIRBuilder.buildInstr(SPIRV::OpDecorate)
.addUse(AsmReg)
.addImm(static_cast<uint32_t>(SPIRV::Decoration::SideEffectsINTEL));
Register DefReg = collectInlineAsmInstrOperands(I2);
if (!DefReg.isValid()) {
DefReg = MRI.createGenericVirtualRegister(LLT::scalar(32));
MRI.setRegClass(DefReg, &SPIRV::iIDRegClass);
SPIRVType *VoidType = GR->getOrCreateSPIRVType(
Type::getVoidTy(MF.getFunction().getContext()), MIRBuilder);
GR->assignSPIRVTypeToVReg(VoidType, DefReg, MF);
}
auto AsmCall = MIRBuilder.buildInstr(SPIRV::OpAsmCallINTEL)
.addDef(DefReg)
.addUse(GR->getSPIRVTypeID(RetType))
.addUse(AsmReg);
for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)
AsmCall.addUse(I1->getOperand(IntrIdx).getReg());
}
for (MachineInstr *MI : ToProcess)
MI->eraseFromParent();
}
static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,
const SPIRVSubtarget &ST,
MachineIRBuilder MIRBuilder) {
SmallVector<MachineInstr *> ToProcess;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (isSpvIntrinsic(MI, Intrinsic::spv_inline_asm) ||
MI.getOpcode() == TargetOpcode::INLINEASM)
ToProcess.push_back(&MI);
}
}
if (ToProcess.size() == 0)
return;
if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_inline_assembly))
report_fatal_error("Inline assembly instructions require the "
"following SPIR-V extension: SPV_INTEL_inline_assembly",
false);
insertInlineAsmProcess(MF, GR, ST, MIRBuilder, ToProcess);
}
static void insertSpirvDecorations(MachineFunction &MF, MachineIRBuilder MIB) {
SmallVector<MachineInstr *, 10> ToErase;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (!isSpvIntrinsic(MI, Intrinsic::spv_assign_decoration))
continue;
MIB.setInsertPt(*MI.getParent(), MI);
buildOpSpirvDecorations(MI.getOperand(1).getReg(), MIB,
MI.getOperand(2).getMetadata());
ToErase.push_back(&MI);
}
}
for (MachineInstr *MI : ToErase)
MI->eraseFromParent();
}
// Find basic blocks of the switch and replace registers in spv_switch() by its
// MBB equivalent.
static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR,
MachineIRBuilder MIB) {
DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;
SmallVector<std::pair<MachineInstr *, SmallVector<MachineInstr *, 8>>>
Switches;
for (MachineBasicBlock &MBB : MF) {
MachineRegisterInfo &MRI = MF.getRegInfo();
BB2MBB[MBB.getBasicBlock()] = &MBB;
for (MachineInstr &MI : MBB) {
if (!isSpvIntrinsic(MI, Intrinsic::spv_switch))
continue;
// Calls to spv_switch intrinsics representing IR switches.
SmallVector<MachineInstr *, 8> NewOps;
for (unsigned i = 2; i < MI.getNumOperands(); ++i) {
Register Reg = MI.getOperand(i).getReg();
if (i % 2 == 1) {
MachineInstr *ConstInstr = getDefInstrMaybeConstant(Reg, &MRI);
NewOps.push_back(ConstInstr);
} else {
MachineInstr *BuildMBB = MRI.getVRegDef(Reg);
assert(BuildMBB &&
BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&
BuildMBB->getOperand(1).isBlockAddress() &&
BuildMBB->getOperand(1).getBlockAddress());
NewOps.push_back(BuildMBB);
}
}
Switches.push_back(std::make_pair(&MI, NewOps));
}
}
SmallPtrSet<MachineInstr *, 8> ToEraseMI;
SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;
for (auto &SwIt : Switches) {
MachineInstr &MI = *SwIt.first;
MachineBasicBlock *MBB = MI.getParent();
SmallVector<MachineInstr *, 8> &Ins = SwIt.second;
SmallVector<MachineOperand, 8> NewOps;
for (unsigned i = 0; i < Ins.size(); ++i) {
if (Ins[i]->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
BasicBlock *CaseBB =
Ins[i]->getOperand(1).getBlockAddress()->getBasicBlock();
auto It = BB2MBB.find(CaseBB);
if (It == BB2MBB.end())
report_fatal_error("cannot find a machine basic block by a basic "
"block in a switch statement");
MachineBasicBlock *Succ = It->second;
ClearAddressTaken.insert(Succ);
NewOps.push_back(MachineOperand::CreateMBB(Succ));
if (!llvm::is_contained(MBB->successors(), Succ))
MBB->addSuccessor(Succ);
ToEraseMI.insert(Ins[i]);
} else {
NewOps.push_back(
MachineOperand::CreateCImm(Ins[i]->getOperand(1).getCImm()));
}
}
for (unsigned i = MI.getNumOperands() - 1; i > 1; --i)
MI.removeOperand(i);
for (auto &MO : NewOps)
MI.addOperand(MO);
if (MachineInstr *Next = MI.getNextNode()) {
if (isSpvIntrinsic(*Next, Intrinsic::spv_track_constant)) {
ToEraseMI.insert(Next);
Next = MI.getNextNode();
}
if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT)
ToEraseMI.insert(Next);
}
}
// If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,
// this leaves their BasicBlock counterparts in a "address taken" status. This
// would make AsmPrinter to generate a series of unneeded labels of a "Address
// of block that was removed by CodeGen" kind. Let's first ensure that we
// don't have a dangling BlockAddress constants by zapping the BlockAddress
// nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions.
Constant *Replacement =
ConstantInt::get(Type::getInt32Ty(MF.getFunction().getContext()), 1);
for (MachineInstr *BlockAddrI : ToEraseMI) {
if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {
BlockAddress *BA = const_cast<BlockAddress *>(
BlockAddrI->getOperand(1).getBlockAddress());
BA->replaceAllUsesWith(
ConstantExpr::getIntToPtr(Replacement, BA->getType()));
BA->destroyConstant();
}
BlockAddrI->eraseFromParent();
}
// BlockAddress operands were used to keep information between passes,
// let's undo the "address taken" status to reflect that Succ doesn't
// actually correspond to an IR-level basic block.
for (MachineBasicBlock *Succ : ClearAddressTaken)
Succ->setAddressTakenIRBlock(nullptr);
}
static bool isImplicitFallthrough(MachineBasicBlock &MBB) {
if (MBB.empty())
return true;
// Branching SPIR-V intrinsics are not detected by this generic method.
// Thus, we can only trust negative result.
if (!MBB.canFallThrough())
return false;
// Otherwise, we must manually check if we have a SPIR-V intrinsic which
// prevent an implicit fallthrough.
for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();
It != E; ++It) {
if (isSpvIntrinsic(*It, Intrinsic::spv_switch))
return false;
}
return true;
}
static void removeImplicitFallthroughs(MachineFunction &MF,
MachineIRBuilder MIB) {
// It is valid for MachineBasicBlocks to not finish with a branch instruction.
// In such cases, they will simply fallthrough their immediate successor.
for (MachineBasicBlock &MBB : MF) {
if (!isImplicitFallthrough(MBB))
continue;
assert(std::distance(MBB.successors().begin(), MBB.successors().end()) ==
1);
MIB.setInsertPt(MBB, MBB.end());
MIB.buildBr(**MBB.successors().begin());
}
}
bool SPIRVPreLegalizer::runOnMachineFunction(MachineFunction &MF) {
// Initialize the type registry.
const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();
SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
GR->setCurrentFunc(MF);
MachineIRBuilder MIB(MF);
// a registry of target extension constants
DenseMap<MachineInstr *, Type *> TargetExtConstTypes;
// to keep record of tracked constants
SmallSet<Register, 4> TrackedConstRegs;
addConstantsToTrack(MF, GR, ST, TargetExtConstTypes, TrackedConstRegs);
foldConstantsIntoIntrinsics(MF, TrackedConstRegs);
insertBitcasts(MF, GR, MIB);
generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes);
processSwitches(MF, GR, MIB);
processInstrsWithTypeFolding(MF, GR, MIB);
removeImplicitFallthroughs(MF, MIB);
insertSpirvDecorations(MF, MIB);
insertInlineAsm(MF, GR, ST, MIB);
return true;
}
INITIALIZE_PASS(SPIRVPreLegalizer, DEBUG_TYPE, "SPIRV pre legalizer", false,
false)
char SPIRVPreLegalizer::ID = 0;
FunctionPass *llvm::createSPIRVPreLegalizerPass() {
return new SPIRVPreLegalizer();
}
Reference in New Issue View Git Blame Copy Permalink