f2ecd86e34
This change removes the uint64_t constructor on LocationSize preventing implicit conversion, and fixes up the using APIs to adapt to the change. Note that I'm adding a couple of explicit conversion points on routines where passing in a fixed offset as an integer seems likely to have well understood semantics. We had an unfortunate case which arose if you tried to pass a TypeSize value to a parameter of LocationSize type. We'd find the implicit conversion path through TypeSize -> uint64_t -> LocationSize which works just fine for fixed values, but looses information and fails assertions if the TypeSize was scalable. This change breaks the first link in that implicit conversion chain since that seemed to be the easier one.
296 lines
10 KiB
C++
296 lines
10 KiB
C++
//===- SIInsertHardClauses.cpp - Insert Hard Clauses ----------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
/// \file
|
|
/// Insert s_clause instructions to form hard clauses.
|
|
///
|
|
/// Clausing load instructions can give cache coherency benefits. Before gfx10,
|
|
/// the hardware automatically detected "soft clauses", which were sequences of
|
|
/// memory instructions of the same type. In gfx10 this detection was removed,
|
|
/// and the s_clause instruction was introduced to explicitly mark "hard
|
|
/// clauses".
|
|
///
|
|
/// It's the scheduler's job to form the clauses by putting similar memory
|
|
/// instructions next to each other. Our job is just to insert an s_clause
|
|
/// instruction to mark the start of each clause.
|
|
///
|
|
/// Note that hard clauses are very similar to, but logically distinct from, the
|
|
/// groups of instructions that have to be restartable when XNACK is enabled.
|
|
/// The rules are slightly different in each case. For example an s_nop
|
|
/// instruction breaks a restartable group, but can appear in the middle of a
|
|
/// hard clause. (Before gfx10 there wasn't a distinction, and both were called
|
|
/// "soft clauses" or just "clauses".)
|
|
///
|
|
/// The SIFormMemoryClauses pass and GCNHazardRecognizer deal with restartable
|
|
/// groups, not hard clauses.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "AMDGPU.h"
|
|
#include "GCNSubtarget.h"
|
|
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
|
#include "llvm/CodeGen/MachinePassManager.h"
|
|
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "si-insert-hard-clauses"
|
|
|
|
namespace {
|
|
|
|
enum HardClauseType {
|
|
// For GFX10:
|
|
|
|
// Texture, buffer, global or scratch memory instructions.
|
|
HARDCLAUSE_VMEM,
|
|
// Flat (not global or scratch) memory instructions.
|
|
HARDCLAUSE_FLAT,
|
|
|
|
// For GFX11:
|
|
|
|
// Texture memory instructions.
|
|
HARDCLAUSE_MIMG_LOAD,
|
|
HARDCLAUSE_MIMG_STORE,
|
|
HARDCLAUSE_MIMG_ATOMIC,
|
|
HARDCLAUSE_MIMG_SAMPLE,
|
|
// Buffer, global or scratch memory instructions.
|
|
HARDCLAUSE_VMEM_LOAD,
|
|
HARDCLAUSE_VMEM_STORE,
|
|
HARDCLAUSE_VMEM_ATOMIC,
|
|
// Flat (not global or scratch) memory instructions.
|
|
HARDCLAUSE_FLAT_LOAD,
|
|
HARDCLAUSE_FLAT_STORE,
|
|
HARDCLAUSE_FLAT_ATOMIC,
|
|
// BVH instructions.
|
|
HARDCLAUSE_BVH,
|
|
|
|
// Common:
|
|
|
|
// Instructions that access LDS.
|
|
HARDCLAUSE_LDS,
|
|
// Scalar memory instructions.
|
|
HARDCLAUSE_SMEM,
|
|
// VALU instructions.
|
|
HARDCLAUSE_VALU,
|
|
LAST_REAL_HARDCLAUSE_TYPE = HARDCLAUSE_VALU,
|
|
|
|
// Internal instructions, which are allowed in the middle of a hard clause,
|
|
// except for s_waitcnt.
|
|
HARDCLAUSE_INTERNAL,
|
|
// Meta instructions that do not result in any ISA like KILL.
|
|
HARDCLAUSE_IGNORE,
|
|
// Instructions that are not allowed in a hard clause: SALU, export, branch,
|
|
// message, GDS, s_waitcnt and anything else not mentioned above.
|
|
HARDCLAUSE_ILLEGAL,
|
|
};
|
|
|
|
class SIInsertHardClauses {
|
|
public:
|
|
const GCNSubtarget *ST = nullptr;
|
|
|
|
HardClauseType getHardClauseType(const MachineInstr &MI) {
|
|
if (MI.mayLoad() || (MI.mayStore() && ST->shouldClusterStores())) {
|
|
if (ST->getGeneration() == AMDGPUSubtarget::GFX10) {
|
|
if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) {
|
|
if (ST->hasNSAClauseBug()) {
|
|
const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode());
|
|
if (Info && Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA)
|
|
return HARDCLAUSE_ILLEGAL;
|
|
}
|
|
return HARDCLAUSE_VMEM;
|
|
}
|
|
if (SIInstrInfo::isFLAT(MI))
|
|
return HARDCLAUSE_FLAT;
|
|
} else {
|
|
assert(ST->getGeneration() >= AMDGPUSubtarget::GFX11);
|
|
if (SIInstrInfo::isMIMG(MI)) {
|
|
const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode());
|
|
const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
|
|
AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
|
|
if (BaseInfo->BVH)
|
|
return HARDCLAUSE_BVH;
|
|
if (BaseInfo->Sampler)
|
|
return HARDCLAUSE_MIMG_SAMPLE;
|
|
return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_MIMG_ATOMIC
|
|
: HARDCLAUSE_MIMG_LOAD
|
|
: HARDCLAUSE_MIMG_STORE;
|
|
}
|
|
if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) {
|
|
return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_VMEM_ATOMIC
|
|
: HARDCLAUSE_VMEM_LOAD
|
|
: HARDCLAUSE_VMEM_STORE;
|
|
}
|
|
if (SIInstrInfo::isFLAT(MI)) {
|
|
return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_FLAT_ATOMIC
|
|
: HARDCLAUSE_FLAT_LOAD
|
|
: HARDCLAUSE_FLAT_STORE;
|
|
}
|
|
}
|
|
// TODO: LDS
|
|
if (SIInstrInfo::isSMRD(MI))
|
|
return HARDCLAUSE_SMEM;
|
|
}
|
|
|
|
// Don't form VALU clauses. It's not clear what benefit they give, if any.
|
|
|
|
// In practice s_nop is the only internal instruction we're likely to see.
|
|
// It's safe to treat the rest as illegal.
|
|
if (MI.getOpcode() == AMDGPU::S_NOP)
|
|
return HARDCLAUSE_INTERNAL;
|
|
if (MI.isMetaInstruction())
|
|
return HARDCLAUSE_IGNORE;
|
|
return HARDCLAUSE_ILLEGAL;
|
|
}
|
|
|
|
// Track information about a clause as we discover it.
|
|
struct ClauseInfo {
|
|
// The type of all (non-internal) instructions in the clause.
|
|
HardClauseType Type = HARDCLAUSE_ILLEGAL;
|
|
// The first (necessarily non-internal) instruction in the clause.
|
|
MachineInstr *First = nullptr;
|
|
// The last non-internal instruction in the clause.
|
|
MachineInstr *Last = nullptr;
|
|
// The length of the clause including any internal instructions in the
|
|
// middle (but not at the end) of the clause.
|
|
unsigned Length = 0;
|
|
// Internal instructions at the and of a clause should not be included in
|
|
// the clause. Count them in TrailingInternalLength until a new memory
|
|
// instruction is added.
|
|
unsigned TrailingInternalLength = 0;
|
|
// The base operands of *Last.
|
|
SmallVector<const MachineOperand *, 4> BaseOps;
|
|
};
|
|
|
|
bool emitClause(const ClauseInfo &CI, const SIInstrInfo *SII) {
|
|
if (CI.First == CI.Last)
|
|
return false;
|
|
assert(CI.Length <= ST->maxHardClauseLength() &&
|
|
"Hard clause is too long!");
|
|
|
|
auto &MBB = *CI.First->getParent();
|
|
auto ClauseMI =
|
|
BuildMI(MBB, *CI.First, DebugLoc(), SII->get(AMDGPU::S_CLAUSE))
|
|
.addImm(CI.Length - 1);
|
|
finalizeBundle(MBB, ClauseMI->getIterator(),
|
|
std::next(CI.Last->getIterator()));
|
|
return true;
|
|
}
|
|
|
|
bool run(MachineFunction &MF) {
|
|
|
|
ST = &MF.getSubtarget<GCNSubtarget>();
|
|
if (!ST->hasHardClauses())
|
|
return false;
|
|
|
|
const SIInstrInfo *SII = ST->getInstrInfo();
|
|
const TargetRegisterInfo *TRI = ST->getRegisterInfo();
|
|
|
|
bool Changed = false;
|
|
for (auto &MBB : MF) {
|
|
ClauseInfo CI;
|
|
for (auto &MI : MBB) {
|
|
HardClauseType Type = getHardClauseType(MI);
|
|
|
|
int64_t Dummy1;
|
|
bool Dummy2;
|
|
LocationSize Dummy3 = LocationSize::precise(0);
|
|
SmallVector<const MachineOperand *, 4> BaseOps;
|
|
if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
|
|
if (!SII->getMemOperandsWithOffsetWidth(MI, BaseOps, Dummy1, Dummy2,
|
|
Dummy3, TRI)) {
|
|
// We failed to get the base operands, so we'll never clause this
|
|
// instruction with any other, so pretend it's illegal.
|
|
Type = HARDCLAUSE_ILLEGAL;
|
|
}
|
|
}
|
|
|
|
if (CI.Length == ST->maxHardClauseLength() ||
|
|
(CI.Length && Type != HARDCLAUSE_INTERNAL &&
|
|
Type != HARDCLAUSE_IGNORE &&
|
|
(Type != CI.Type ||
|
|
// Note that we lie to shouldClusterMemOps about the size of the
|
|
// cluster. When shouldClusterMemOps is called from the machine
|
|
// scheduler it limits the size of the cluster to avoid increasing
|
|
// register pressure too much, but this pass runs after register
|
|
// allocation so there is no need for that kind of limit.
|
|
// We also lie about the Offset and OffsetIsScalable parameters,
|
|
// as they aren't used in the SIInstrInfo implementation.
|
|
!SII->shouldClusterMemOps(CI.BaseOps, 0, false, BaseOps, 0, false,
|
|
2, 2)))) {
|
|
// Finish the current clause.
|
|
Changed |= emitClause(CI, SII);
|
|
CI = ClauseInfo();
|
|
}
|
|
|
|
if (CI.Length) {
|
|
// Extend the current clause.
|
|
if (Type != HARDCLAUSE_IGNORE) {
|
|
if (Type == HARDCLAUSE_INTERNAL) {
|
|
++CI.TrailingInternalLength;
|
|
} else {
|
|
++CI.Length;
|
|
CI.Length += CI.TrailingInternalLength;
|
|
CI.TrailingInternalLength = 0;
|
|
CI.Last = &MI;
|
|
CI.BaseOps = std::move(BaseOps);
|
|
}
|
|
}
|
|
} else if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
|
|
// Start a new clause.
|
|
CI = ClauseInfo{Type, &MI, &MI, 1, 0, std::move(BaseOps)};
|
|
}
|
|
}
|
|
|
|
// Finish the last clause in the basic block if any.
|
|
if (CI.Length)
|
|
Changed |= emitClause(CI, SII);
|
|
}
|
|
|
|
return Changed;
|
|
}
|
|
};
|
|
|
|
class SIInsertHardClausesLegacy : public MachineFunctionPass {
|
|
public:
|
|
static char ID;
|
|
SIInsertHardClausesLegacy() : MachineFunctionPass(ID) {}
|
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override {
|
|
if (skipFunction(MF.getFunction()))
|
|
return false;
|
|
|
|
return SIInsertHardClauses().run(MF);
|
|
}
|
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
|
AU.setPreservesCFG();
|
|
MachineFunctionPass::getAnalysisUsage(AU);
|
|
}
|
|
};
|
|
|
|
} // namespace
|
|
|
|
PreservedAnalyses
|
|
llvm::SIInsertHardClausesPass::run(MachineFunction &MF,
|
|
MachineFunctionAnalysisManager &MFAM) {
|
|
if (!SIInsertHardClauses().run(MF))
|
|
return PreservedAnalyses::all();
|
|
|
|
auto PA = getMachineFunctionPassPreservedAnalyses();
|
|
PA.preserveSet<CFGAnalyses>();
|
|
return PA;
|
|
}
|
|
|
|
char SIInsertHardClausesLegacy::ID = 0;
|
|
|
|
char &llvm::SIInsertHardClausesID = SIInsertHardClausesLegacy::ID;
|
|
|
|
INITIALIZE_PASS(SIInsertHardClausesLegacy, DEBUG_TYPE, "SI Insert Hard Clauses",
|
|
false, false)
|