faa2c678aa
In order to enable the LLVM frontend to better analyze buffer operations (and to potentially enable more precise analyses on the backend), define versions of the raw and structured buffer intrinsics that use `ptr addrspace(8)` instead of `<4 x i32>` to represent their rsrc arguments. The new intrinsics are named by replacing `buffer.` with `buffer.ptr`. One advantage to these intrinsic definitions is that, instead of specifying that a buffer load/store will read/write some memory, we can indicate that the memory read or written will be based on the pointer argument. This means that, for example, a read from a `noalias` buffer can be pulled out of a loop that is modifying a distinct buffer. In the future, we will define custom PseudoSourceValues that will allow us to package up the (buffer, index, offset) triples that buffer intrinsics contain and allow for more precise backend analysis. This work also enables creating address space 7, which represents manipulation of raw buffers using native LLVM load and store instructions. Where tests simply used a buffer intrinsic while testing some other code path (such as the tests for VGPR spills), they have been updated to use the new intrinsic form. Tests that are "about" buffer intrinsics (for instance, those that ensure that they codegen as expected) have been duplicated, either within existing files or into new ones. Depends on D145441 Reviewed By: arsenm, #amdgpu Differential Revision: https://reviews.llvm.org/D147547
777 lines
28 KiB
C++
777 lines
28 KiB
C++
//===-- AMDGPUAtomicOptimizer.cpp -----------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
/// \file
|
|
/// This pass optimizes atomic operations by using a single lane of a wavefront
|
|
/// to perform the atomic operation, thus reducing contention on that memory
|
|
/// location.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "AMDGPU.h"
|
|
#include "GCNSubtarget.h"
|
|
#include "llvm/Analysis/DomTreeUpdater.h"
|
|
#include "llvm/Analysis/UniformityAnalysis.h"
|
|
#include "llvm/CodeGen/TargetPassConfig.h"
|
|
#include "llvm/IR/IRBuilder.h"
|
|
#include "llvm/IR/InstVisitor.h"
|
|
#include "llvm/IR/IntrinsicsAMDGPU.h"
|
|
#include "llvm/InitializePasses.h"
|
|
#include "llvm/Target/TargetMachine.h"
|
|
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
|
|
|
|
#define DEBUG_TYPE "amdgpu-atomic-optimizer"
|
|
|
|
using namespace llvm;
|
|
using namespace llvm::AMDGPU;
|
|
|
|
namespace {
|
|
|
|
struct ReplacementInfo {
|
|
Instruction *I;
|
|
AtomicRMWInst::BinOp Op;
|
|
unsigned ValIdx;
|
|
bool ValDivergent;
|
|
};
|
|
|
|
class AMDGPUAtomicOptimizer : public FunctionPass {
|
|
public:
|
|
static char ID;
|
|
|
|
AMDGPUAtomicOptimizer() : FunctionPass(ID) {}
|
|
|
|
bool runOnFunction(Function &F) override;
|
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
|
AU.addPreserved<DominatorTreeWrapperPass>();
|
|
AU.addRequired<UniformityInfoWrapperPass>();
|
|
AU.addRequired<TargetPassConfig>();
|
|
}
|
|
};
|
|
|
|
class AMDGPUAtomicOptimizerImpl
|
|
: public InstVisitor<AMDGPUAtomicOptimizerImpl> {
|
|
private:
|
|
SmallVector<ReplacementInfo, 8> ToReplace;
|
|
const UniformityInfo *UA;
|
|
const DataLayout *DL;
|
|
DomTreeUpdater &DTU;
|
|
const GCNSubtarget *ST;
|
|
bool IsPixelShader;
|
|
|
|
Value *buildReduction(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
|
|
Value *const Identity) const;
|
|
Value *buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
|
|
Value *const Identity) const;
|
|
Value *buildShiftRight(IRBuilder<> &B, Value *V, Value *const Identity) const;
|
|
|
|
void optimizeAtomic(Instruction &I, AtomicRMWInst::BinOp Op, unsigned ValIdx,
|
|
bool ValDivergent) const;
|
|
|
|
public:
|
|
AMDGPUAtomicOptimizerImpl() = delete;
|
|
|
|
AMDGPUAtomicOptimizerImpl(const UniformityInfo *UA, const DataLayout *DL,
|
|
DomTreeUpdater &DTU, const GCNSubtarget *ST,
|
|
bool IsPixelShader)
|
|
: UA(UA), DL(DL), DTU(DTU), ST(ST), IsPixelShader(IsPixelShader) {}
|
|
|
|
bool run(Function &F);
|
|
|
|
void visitAtomicRMWInst(AtomicRMWInst &I);
|
|
void visitIntrinsicInst(IntrinsicInst &I);
|
|
};
|
|
|
|
} // namespace
|
|
|
|
char AMDGPUAtomicOptimizer::ID = 0;
|
|
|
|
char &llvm::AMDGPUAtomicOptimizerID = AMDGPUAtomicOptimizer::ID;
|
|
|
|
bool AMDGPUAtomicOptimizer::runOnFunction(Function &F) {
|
|
if (skipFunction(F)) {
|
|
return false;
|
|
}
|
|
|
|
const UniformityInfo *UA =
|
|
&getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
|
|
const DataLayout *DL = &F.getParent()->getDataLayout();
|
|
|
|
DominatorTreeWrapperPass *const DTW =
|
|
getAnalysisIfAvailable<DominatorTreeWrapperPass>();
|
|
DomTreeUpdater DTU(DTW ? &DTW->getDomTree() : nullptr,
|
|
DomTreeUpdater::UpdateStrategy::Lazy);
|
|
|
|
const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
|
|
const TargetMachine &TM = TPC.getTM<TargetMachine>();
|
|
const GCNSubtarget *ST = &TM.getSubtarget<GCNSubtarget>(F);
|
|
|
|
bool IsPixelShader = F.getCallingConv() == CallingConv::AMDGPU_PS;
|
|
|
|
return AMDGPUAtomicOptimizerImpl(UA, DL, DTU, ST, IsPixelShader).run(F);
|
|
}
|
|
|
|
PreservedAnalyses AMDGPUAtomicOptimizerPass::run(Function &F,
|
|
FunctionAnalysisManager &AM) {
|
|
|
|
const auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
|
|
const DataLayout *DL = &F.getParent()->getDataLayout();
|
|
|
|
DomTreeUpdater DTU(&AM.getResult<DominatorTreeAnalysis>(F),
|
|
DomTreeUpdater::UpdateStrategy::Lazy);
|
|
const GCNSubtarget *ST = &TM.getSubtarget<GCNSubtarget>(F);
|
|
|
|
bool IsPixelShader = F.getCallingConv() == CallingConv::AMDGPU_PS;
|
|
|
|
return AMDGPUAtomicOptimizerImpl(UA, DL, DTU, ST, IsPixelShader).run(F)
|
|
? PreservedAnalyses::none()
|
|
: PreservedAnalyses::all();
|
|
}
|
|
|
|
bool AMDGPUAtomicOptimizerImpl::run(Function &F) {
|
|
|
|
visit(F);
|
|
|
|
const bool Changed = !ToReplace.empty();
|
|
|
|
for (ReplacementInfo &Info : ToReplace) {
|
|
optimizeAtomic(*Info.I, Info.Op, Info.ValIdx, Info.ValDivergent);
|
|
}
|
|
|
|
ToReplace.clear();
|
|
|
|
return Changed;
|
|
}
|
|
|
|
void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
|
|
// Early exit for unhandled address space atomic instructions.
|
|
switch (I.getPointerAddressSpace()) {
|
|
default:
|
|
return;
|
|
case AMDGPUAS::GLOBAL_ADDRESS:
|
|
case AMDGPUAS::LOCAL_ADDRESS:
|
|
break;
|
|
}
|
|
|
|
AtomicRMWInst::BinOp Op = I.getOperation();
|
|
|
|
switch (Op) {
|
|
default:
|
|
return;
|
|
case AtomicRMWInst::Add:
|
|
case AtomicRMWInst::Sub:
|
|
case AtomicRMWInst::And:
|
|
case AtomicRMWInst::Or:
|
|
case AtomicRMWInst::Xor:
|
|
case AtomicRMWInst::Max:
|
|
case AtomicRMWInst::Min:
|
|
case AtomicRMWInst::UMax:
|
|
case AtomicRMWInst::UMin:
|
|
break;
|
|
}
|
|
|
|
const unsigned PtrIdx = 0;
|
|
const unsigned ValIdx = 1;
|
|
|
|
// If the pointer operand is divergent, then each lane is doing an atomic
|
|
// operation on a different address, and we cannot optimize that.
|
|
if (UA->isDivergentUse(I.getOperandUse(PtrIdx))) {
|
|
return;
|
|
}
|
|
|
|
const bool ValDivergent = UA->isDivergentUse(I.getOperandUse(ValIdx));
|
|
|
|
// If the value operand is divergent, each lane is contributing a different
|
|
// value to the atomic calculation. We can only optimize divergent values if
|
|
// we have DPP available on our subtarget, and the atomic operation is 32
|
|
// bits.
|
|
if (ValDivergent &&
|
|
(!ST->hasDPP() || DL->getTypeSizeInBits(I.getType()) != 32)) {
|
|
return;
|
|
}
|
|
|
|
// If we get here, we can optimize the atomic using a single wavefront-wide
|
|
// atomic operation to do the calculation for the entire wavefront, so
|
|
// remember the instruction so we can come back to it.
|
|
const ReplacementInfo Info = {&I, Op, ValIdx, ValDivergent};
|
|
|
|
ToReplace.push_back(Info);
|
|
}
|
|
|
|
void AMDGPUAtomicOptimizerImpl::visitIntrinsicInst(IntrinsicInst &I) {
|
|
AtomicRMWInst::BinOp Op;
|
|
|
|
switch (I.getIntrinsicID()) {
|
|
default:
|
|
return;
|
|
case Intrinsic::amdgcn_buffer_atomic_add:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_add:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_add:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
|
|
Op = AtomicRMWInst::Add;
|
|
break;
|
|
case Intrinsic::amdgcn_buffer_atomic_sub:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_sub:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_sub:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
|
|
Op = AtomicRMWInst::Sub;
|
|
break;
|
|
case Intrinsic::amdgcn_buffer_atomic_and:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_and:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_and:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
|
|
Op = AtomicRMWInst::And;
|
|
break;
|
|
case Intrinsic::amdgcn_buffer_atomic_or:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_or:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_or:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
|
|
Op = AtomicRMWInst::Or;
|
|
break;
|
|
case Intrinsic::amdgcn_buffer_atomic_xor:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_xor:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_xor:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
|
|
Op = AtomicRMWInst::Xor;
|
|
break;
|
|
case Intrinsic::amdgcn_buffer_atomic_smin:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_smin:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_smin:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
|
|
Op = AtomicRMWInst::Min;
|
|
break;
|
|
case Intrinsic::amdgcn_buffer_atomic_umin:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_umin:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_umin:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
|
|
Op = AtomicRMWInst::UMin;
|
|
break;
|
|
case Intrinsic::amdgcn_buffer_atomic_smax:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_smax:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_smax:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
|
|
Op = AtomicRMWInst::Max;
|
|
break;
|
|
case Intrinsic::amdgcn_buffer_atomic_umax:
|
|
case Intrinsic::amdgcn_struct_buffer_atomic_umax:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
|
|
case Intrinsic::amdgcn_raw_buffer_atomic_umax:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
|
|
Op = AtomicRMWInst::UMax;
|
|
break;
|
|
}
|
|
|
|
const unsigned ValIdx = 0;
|
|
|
|
const bool ValDivergent = UA->isDivergentUse(I.getOperandUse(ValIdx));
|
|
|
|
// If the value operand is divergent, each lane is contributing a different
|
|
// value to the atomic calculation. We can only optimize divergent values if
|
|
// we have DPP available on our subtarget, and the atomic operation is 32
|
|
// bits.
|
|
if (ValDivergent &&
|
|
(!ST->hasDPP() || DL->getTypeSizeInBits(I.getType()) != 32)) {
|
|
return;
|
|
}
|
|
|
|
// If any of the other arguments to the intrinsic are divergent, we can't
|
|
// optimize the operation.
|
|
for (unsigned Idx = 1; Idx < I.getNumOperands(); Idx++) {
|
|
if (UA->isDivergentUse(I.getOperandUse(Idx))) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
// If we get here, we can optimize the atomic using a single wavefront-wide
|
|
// atomic operation to do the calculation for the entire wavefront, so
|
|
// remember the instruction so we can come back to it.
|
|
const ReplacementInfo Info = {&I, Op, ValIdx, ValDivergent};
|
|
|
|
ToReplace.push_back(Info);
|
|
}
|
|
|
|
// Use the builder to create the non-atomic counterpart of the specified
|
|
// atomicrmw binary op.
|
|
static Value *buildNonAtomicBinOp(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
|
|
Value *LHS, Value *RHS) {
|
|
CmpInst::Predicate Pred;
|
|
|
|
switch (Op) {
|
|
default:
|
|
llvm_unreachable("Unhandled atomic op");
|
|
case AtomicRMWInst::Add:
|
|
return B.CreateBinOp(Instruction::Add, LHS, RHS);
|
|
case AtomicRMWInst::Sub:
|
|
return B.CreateBinOp(Instruction::Sub, LHS, RHS);
|
|
case AtomicRMWInst::And:
|
|
return B.CreateBinOp(Instruction::And, LHS, RHS);
|
|
case AtomicRMWInst::Or:
|
|
return B.CreateBinOp(Instruction::Or, LHS, RHS);
|
|
case AtomicRMWInst::Xor:
|
|
return B.CreateBinOp(Instruction::Xor, LHS, RHS);
|
|
|
|
case AtomicRMWInst::Max:
|
|
Pred = CmpInst::ICMP_SGT;
|
|
break;
|
|
case AtomicRMWInst::Min:
|
|
Pred = CmpInst::ICMP_SLT;
|
|
break;
|
|
case AtomicRMWInst::UMax:
|
|
Pred = CmpInst::ICMP_UGT;
|
|
break;
|
|
case AtomicRMWInst::UMin:
|
|
Pred = CmpInst::ICMP_ULT;
|
|
break;
|
|
}
|
|
Value *Cond = B.CreateICmp(Pred, LHS, RHS);
|
|
return B.CreateSelect(Cond, LHS, RHS);
|
|
}
|
|
|
|
// Use the builder to create a reduction of V across the wavefront, with all
|
|
// lanes active, returning the same result in all lanes.
|
|
Value *AMDGPUAtomicOptimizerImpl::buildReduction(IRBuilder<> &B,
|
|
AtomicRMWInst::BinOp Op,
|
|
Value *V,
|
|
Value *const Identity) const {
|
|
Type *const Ty = V->getType();
|
|
Module *M = B.GetInsertBlock()->getModule();
|
|
Function *UpdateDPP =
|
|
Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, Ty);
|
|
|
|
// Reduce within each row of 16 lanes.
|
|
for (unsigned Idx = 0; Idx < 4; Idx++) {
|
|
V = buildNonAtomicBinOp(
|
|
B, Op, V,
|
|
B.CreateCall(UpdateDPP,
|
|
{Identity, V, B.getInt32(DPP::ROW_XMASK0 | 1 << Idx),
|
|
B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}));
|
|
}
|
|
|
|
// Reduce within each pair of rows (i.e. 32 lanes).
|
|
assert(ST->hasPermLaneX16());
|
|
V = buildNonAtomicBinOp(
|
|
B, Op, V,
|
|
B.CreateIntrinsic(
|
|
Intrinsic::amdgcn_permlanex16, {},
|
|
{V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()}));
|
|
|
|
if (ST->isWave32())
|
|
return V;
|
|
|
|
if (ST->hasPermLane64()) {
|
|
// Reduce across the upper and lower 32 lanes.
|
|
return buildNonAtomicBinOp(
|
|
B, Op, V, B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {}, V));
|
|
}
|
|
|
|
// Pick an arbitrary lane from 0..31 and an arbitrary lane from 32..63 and
|
|
// combine them with a scalar operation.
|
|
Function *ReadLane =
|
|
Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, {});
|
|
Value *const Lane0 = B.CreateCall(ReadLane, {V, B.getInt32(0)});
|
|
Value *const Lane32 = B.CreateCall(ReadLane, {V, B.getInt32(32)});
|
|
return buildNonAtomicBinOp(B, Op, Lane0, Lane32);
|
|
}
|
|
|
|
// Use the builder to create an inclusive scan of V across the wavefront, with
|
|
// all lanes active.
|
|
Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B,
|
|
AtomicRMWInst::BinOp Op, Value *V,
|
|
Value *const Identity) const {
|
|
Type *const Ty = V->getType();
|
|
Module *M = B.GetInsertBlock()->getModule();
|
|
Function *UpdateDPP =
|
|
Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, Ty);
|
|
|
|
for (unsigned Idx = 0; Idx < 4; Idx++) {
|
|
V = buildNonAtomicBinOp(
|
|
B, Op, V,
|
|
B.CreateCall(UpdateDPP,
|
|
{Identity, V, B.getInt32(DPP::ROW_SHR0 | 1 << Idx),
|
|
B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}));
|
|
}
|
|
if (ST->hasDPPBroadcasts()) {
|
|
// GFX9 has DPP row broadcast operations.
|
|
V = buildNonAtomicBinOp(
|
|
B, Op, V,
|
|
B.CreateCall(UpdateDPP,
|
|
{Identity, V, B.getInt32(DPP::BCAST15), B.getInt32(0xa),
|
|
B.getInt32(0xf), B.getFalse()}));
|
|
V = buildNonAtomicBinOp(
|
|
B, Op, V,
|
|
B.CreateCall(UpdateDPP,
|
|
{Identity, V, B.getInt32(DPP::BCAST31), B.getInt32(0xc),
|
|
B.getInt32(0xf), B.getFalse()}));
|
|
} else {
|
|
// On GFX10 all DPP operations are confined to a single row. To get cross-
|
|
// row operations we have to use permlane or readlane.
|
|
|
|
// Combine lane 15 into lanes 16..31 (and, for wave 64, lane 47 into lanes
|
|
// 48..63).
|
|
assert(ST->hasPermLaneX16());
|
|
Value *const PermX = B.CreateIntrinsic(
|
|
Intrinsic::amdgcn_permlanex16, {},
|
|
{V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()});
|
|
V = buildNonAtomicBinOp(
|
|
B, Op, V,
|
|
B.CreateCall(UpdateDPP,
|
|
{Identity, PermX, B.getInt32(DPP::QUAD_PERM_ID),
|
|
B.getInt32(0xa), B.getInt32(0xf), B.getFalse()}));
|
|
if (!ST->isWave32()) {
|
|
// Combine lane 31 into lanes 32..63.
|
|
Value *const Lane31 = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {},
|
|
{V, B.getInt32(31)});
|
|
V = buildNonAtomicBinOp(
|
|
B, Op, V,
|
|
B.CreateCall(UpdateDPP,
|
|
{Identity, Lane31, B.getInt32(DPP::QUAD_PERM_ID),
|
|
B.getInt32(0xc), B.getInt32(0xf), B.getFalse()}));
|
|
}
|
|
}
|
|
return V;
|
|
}
|
|
|
|
// Use the builder to create a shift right of V across the wavefront, with all
|
|
// lanes active, to turn an inclusive scan into an exclusive scan.
|
|
Value *AMDGPUAtomicOptimizerImpl::buildShiftRight(IRBuilder<> &B, Value *V,
|
|
Value *const Identity) const {
|
|
Type *const Ty = V->getType();
|
|
Module *M = B.GetInsertBlock()->getModule();
|
|
Function *UpdateDPP =
|
|
Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, Ty);
|
|
|
|
if (ST->hasDPPWavefrontShifts()) {
|
|
// GFX9 has DPP wavefront shift operations.
|
|
V = B.CreateCall(UpdateDPP,
|
|
{Identity, V, B.getInt32(DPP::WAVE_SHR1), B.getInt32(0xf),
|
|
B.getInt32(0xf), B.getFalse()});
|
|
} else {
|
|
Function *ReadLane =
|
|
Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, {});
|
|
Function *WriteLane =
|
|
Intrinsic::getDeclaration(M, Intrinsic::amdgcn_writelane, {});
|
|
|
|
// On GFX10 all DPP operations are confined to a single row. To get cross-
|
|
// row operations we have to use permlane or readlane.
|
|
Value *Old = V;
|
|
V = B.CreateCall(UpdateDPP,
|
|
{Identity, V, B.getInt32(DPP::ROW_SHR0 + 1),
|
|
B.getInt32(0xf), B.getInt32(0xf), B.getFalse()});
|
|
|
|
// Copy the old lane 15 to the new lane 16.
|
|
V = B.CreateCall(WriteLane, {B.CreateCall(ReadLane, {Old, B.getInt32(15)}),
|
|
B.getInt32(16), V});
|
|
|
|
if (!ST->isWave32()) {
|
|
// Copy the old lane 31 to the new lane 32.
|
|
V = B.CreateCall(
|
|
WriteLane,
|
|
{B.CreateCall(ReadLane, {Old, B.getInt32(31)}), B.getInt32(32), V});
|
|
|
|
// Copy the old lane 47 to the new lane 48.
|
|
V = B.CreateCall(
|
|
WriteLane,
|
|
{B.CreateCall(ReadLane, {Old, B.getInt32(47)}), B.getInt32(48), V});
|
|
}
|
|
}
|
|
|
|
return V;
|
|
}
|
|
|
|
static APInt getIdentityValueForAtomicOp(AtomicRMWInst::BinOp Op,
|
|
unsigned BitWidth) {
|
|
switch (Op) {
|
|
default:
|
|
llvm_unreachable("Unhandled atomic op");
|
|
case AtomicRMWInst::Add:
|
|
case AtomicRMWInst::Sub:
|
|
case AtomicRMWInst::Or:
|
|
case AtomicRMWInst::Xor:
|
|
case AtomicRMWInst::UMax:
|
|
return APInt::getMinValue(BitWidth);
|
|
case AtomicRMWInst::And:
|
|
case AtomicRMWInst::UMin:
|
|
return APInt::getMaxValue(BitWidth);
|
|
case AtomicRMWInst::Max:
|
|
return APInt::getSignedMinValue(BitWidth);
|
|
case AtomicRMWInst::Min:
|
|
return APInt::getSignedMaxValue(BitWidth);
|
|
}
|
|
}
|
|
|
|
static Value *buildMul(IRBuilder<> &B, Value *LHS, Value *RHS) {
|
|
const ConstantInt *CI = dyn_cast<ConstantInt>(LHS);
|
|
return (CI && CI->isOne()) ? RHS : B.CreateMul(LHS, RHS);
|
|
}
|
|
|
|
void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
|
|
AtomicRMWInst::BinOp Op,
|
|
unsigned ValIdx,
|
|
bool ValDivergent) const {
|
|
// Start building just before the instruction.
|
|
IRBuilder<> B(&I);
|
|
|
|
// If we are in a pixel shader, because of how we have to mask out helper
|
|
// lane invocations, we need to record the entry and exit BB's.
|
|
BasicBlock *PixelEntryBB = nullptr;
|
|
BasicBlock *PixelExitBB = nullptr;
|
|
|
|
// If we're optimizing an atomic within a pixel shader, we need to wrap the
|
|
// entire atomic operation in a helper-lane check. We do not want any helper
|
|
// lanes that are around only for the purposes of derivatives to take part
|
|
// in any cross-lane communication, and we use a branch on whether the lane is
|
|
// live to do this.
|
|
if (IsPixelShader) {
|
|
// Record I's original position as the entry block.
|
|
PixelEntryBB = I.getParent();
|
|
|
|
Value *const Cond = B.CreateIntrinsic(Intrinsic::amdgcn_ps_live, {}, {});
|
|
Instruction *const NonHelperTerminator =
|
|
SplitBlockAndInsertIfThen(Cond, &I, false, nullptr, &DTU, nullptr);
|
|
|
|
// Record I's new position as the exit block.
|
|
PixelExitBB = I.getParent();
|
|
|
|
I.moveBefore(NonHelperTerminator);
|
|
B.SetInsertPoint(&I);
|
|
}
|
|
|
|
Type *const Ty = I.getType();
|
|
const unsigned TyBitWidth = DL->getTypeSizeInBits(Ty);
|
|
auto *const VecTy = FixedVectorType::get(B.getInt32Ty(), 2);
|
|
|
|
// This is the value in the atomic operation we need to combine in order to
|
|
// reduce the number of atomic operations.
|
|
Value *const V = I.getOperand(ValIdx);
|
|
|
|
// We need to know how many lanes are active within the wavefront, and we do
|
|
// this by doing a ballot of active lanes.
|
|
Type *const WaveTy = B.getIntNTy(ST->getWavefrontSize());
|
|
CallInst *const Ballot =
|
|
B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue());
|
|
|
|
// We need to know how many lanes are active within the wavefront that are
|
|
// below us. If we counted each lane linearly starting from 0, a lane is
|
|
// below us only if its associated index was less than ours. We do this by
|
|
// using the mbcnt intrinsic.
|
|
Value *Mbcnt;
|
|
if (ST->isWave32()) {
|
|
Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {},
|
|
{Ballot, B.getInt32(0)});
|
|
} else {
|
|
Value *const BitCast = B.CreateBitCast(Ballot, VecTy);
|
|
Value *const ExtractLo = B.CreateExtractElement(BitCast, B.getInt32(0));
|
|
Value *const ExtractHi = B.CreateExtractElement(BitCast, B.getInt32(1));
|
|
Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {},
|
|
{ExtractLo, B.getInt32(0)});
|
|
Mbcnt =
|
|
B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {}, {ExtractHi, Mbcnt});
|
|
}
|
|
Mbcnt = B.CreateIntCast(Mbcnt, Ty, false);
|
|
|
|
Value *const Identity = B.getInt(getIdentityValueForAtomicOp(Op, TyBitWidth));
|
|
|
|
Value *ExclScan = nullptr;
|
|
Value *NewV = nullptr;
|
|
|
|
const bool NeedResult = !I.use_empty();
|
|
|
|
// If we have a divergent value in each lane, we need to combine the value
|
|
// using DPP.
|
|
if (ValDivergent) {
|
|
// First we need to set all inactive invocations to the identity value, so
|
|
// that they can correctly contribute to the final result.
|
|
NewV = B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, Ty, {V, Identity});
|
|
|
|
const AtomicRMWInst::BinOp ScanOp =
|
|
Op == AtomicRMWInst::Sub ? AtomicRMWInst::Add : Op;
|
|
if (!NeedResult && ST->hasPermLaneX16()) {
|
|
// On GFX10 the permlanex16 instruction helps us build a reduction without
|
|
// too many readlanes and writelanes, which are generally bad for
|
|
// performance.
|
|
NewV = buildReduction(B, ScanOp, NewV, Identity);
|
|
} else {
|
|
NewV = buildScan(B, ScanOp, NewV, Identity);
|
|
if (NeedResult)
|
|
ExclScan = buildShiftRight(B, NewV, Identity);
|
|
|
|
// Read the value from the last lane, which has accumulated the values of
|
|
// each active lane in the wavefront. This will be our new value which we
|
|
// will provide to the atomic operation.
|
|
Value *const LastLaneIdx = B.getInt32(ST->getWavefrontSize() - 1);
|
|
assert(TyBitWidth == 32);
|
|
NewV = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {},
|
|
{NewV, LastLaneIdx});
|
|
}
|
|
|
|
// Finally mark the readlanes in the WWM section.
|
|
NewV = B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, NewV);
|
|
} else {
|
|
switch (Op) {
|
|
default:
|
|
llvm_unreachable("Unhandled atomic op");
|
|
|
|
case AtomicRMWInst::Add:
|
|
case AtomicRMWInst::Sub: {
|
|
// The new value we will be contributing to the atomic operation is the
|
|
// old value times the number of active lanes.
|
|
Value *const Ctpop = B.CreateIntCast(
|
|
B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false);
|
|
NewV = buildMul(B, V, Ctpop);
|
|
break;
|
|
}
|
|
|
|
case AtomicRMWInst::And:
|
|
case AtomicRMWInst::Or:
|
|
case AtomicRMWInst::Max:
|
|
case AtomicRMWInst::Min:
|
|
case AtomicRMWInst::UMax:
|
|
case AtomicRMWInst::UMin:
|
|
// These operations with a uniform value are idempotent: doing the atomic
|
|
// operation multiple times has the same effect as doing it once.
|
|
NewV = V;
|
|
break;
|
|
|
|
case AtomicRMWInst::Xor:
|
|
// The new value we will be contributing to the atomic operation is the
|
|
// old value times the parity of the number of active lanes.
|
|
Value *const Ctpop = B.CreateIntCast(
|
|
B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false);
|
|
NewV = buildMul(B, V, B.CreateAnd(Ctpop, 1));
|
|
break;
|
|
}
|
|
}
|
|
|
|
// We only want a single lane to enter our new control flow, and we do this
|
|
// by checking if there are any active lanes below us. Only one lane will
|
|
// have 0 active lanes below us, so that will be the only one to progress.
|
|
Value *const Cond = B.CreateICmpEQ(Mbcnt, B.getIntN(TyBitWidth, 0));
|
|
|
|
// Store I's original basic block before we split the block.
|
|
BasicBlock *const EntryBB = I.getParent();
|
|
|
|
// We need to introduce some new control flow to force a single lane to be
|
|
// active. We do this by splitting I's basic block at I, and introducing the
|
|
// new block such that:
|
|
// entry --> single_lane -\
|
|
// \------------------> exit
|
|
Instruction *const SingleLaneTerminator =
|
|
SplitBlockAndInsertIfThen(Cond, &I, false, nullptr, &DTU, nullptr);
|
|
|
|
// Move the IR builder into single_lane next.
|
|
B.SetInsertPoint(SingleLaneTerminator);
|
|
|
|
// Clone the original atomic operation into single lane, replacing the
|
|
// original value with our newly created one.
|
|
Instruction *const NewI = I.clone();
|
|
B.Insert(NewI);
|
|
NewI->setOperand(ValIdx, NewV);
|
|
|
|
// Move the IR builder into exit next, and start inserting just before the
|
|
// original instruction.
|
|
B.SetInsertPoint(&I);
|
|
|
|
if (NeedResult) {
|
|
// Create a PHI node to get our new atomic result into the exit block.
|
|
PHINode *const PHI = B.CreatePHI(Ty, 2);
|
|
PHI->addIncoming(PoisonValue::get(Ty), EntryBB);
|
|
PHI->addIncoming(NewI, SingleLaneTerminator->getParent());
|
|
|
|
// We need to broadcast the value who was the lowest active lane (the first
|
|
// lane) to all other lanes in the wavefront. We use an intrinsic for this,
|
|
// but have to handle 64-bit broadcasts with two calls to this intrinsic.
|
|
Value *BroadcastI = nullptr;
|
|
|
|
if (TyBitWidth == 64) {
|
|
Value *const ExtractLo = B.CreateTrunc(PHI, B.getInt32Ty());
|
|
Value *const ExtractHi =
|
|
B.CreateTrunc(B.CreateLShr(PHI, 32), B.getInt32Ty());
|
|
CallInst *const ReadFirstLaneLo =
|
|
B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo);
|
|
CallInst *const ReadFirstLaneHi =
|
|
B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractHi);
|
|
Value *const PartialInsert = B.CreateInsertElement(
|
|
PoisonValue::get(VecTy), ReadFirstLaneLo, B.getInt32(0));
|
|
Value *const Insert =
|
|
B.CreateInsertElement(PartialInsert, ReadFirstLaneHi, B.getInt32(1));
|
|
BroadcastI = B.CreateBitCast(Insert, Ty);
|
|
} else if (TyBitWidth == 32) {
|
|
|
|
BroadcastI = B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, PHI);
|
|
} else {
|
|
llvm_unreachable("Unhandled atomic bit width");
|
|
}
|
|
|
|
// Now that we have the result of our single atomic operation, we need to
|
|
// get our individual lane's slice into the result. We use the lane offset
|
|
// we previously calculated combined with the atomic result value we got
|
|
// from the first lane, to get our lane's index into the atomic result.
|
|
Value *LaneOffset = nullptr;
|
|
if (ValDivergent) {
|
|
LaneOffset =
|
|
B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, ExclScan);
|
|
} else {
|
|
switch (Op) {
|
|
default:
|
|
llvm_unreachable("Unhandled atomic op");
|
|
case AtomicRMWInst::Add:
|
|
case AtomicRMWInst::Sub:
|
|
LaneOffset = buildMul(B, V, Mbcnt);
|
|
break;
|
|
case AtomicRMWInst::And:
|
|
case AtomicRMWInst::Or:
|
|
case AtomicRMWInst::Max:
|
|
case AtomicRMWInst::Min:
|
|
case AtomicRMWInst::UMax:
|
|
case AtomicRMWInst::UMin:
|
|
LaneOffset = B.CreateSelect(Cond, Identity, V);
|
|
break;
|
|
case AtomicRMWInst::Xor:
|
|
LaneOffset = buildMul(B, V, B.CreateAnd(Mbcnt, 1));
|
|
break;
|
|
}
|
|
}
|
|
Value *const Result = buildNonAtomicBinOp(B, Op, BroadcastI, LaneOffset);
|
|
|
|
if (IsPixelShader) {
|
|
// Need a final PHI to reconverge to above the helper lane branch mask.
|
|
B.SetInsertPoint(PixelExitBB->getFirstNonPHI());
|
|
|
|
PHINode *const PHI = B.CreatePHI(Ty, 2);
|
|
PHI->addIncoming(PoisonValue::get(Ty), PixelEntryBB);
|
|
PHI->addIncoming(Result, I.getParent());
|
|
I.replaceAllUsesWith(PHI);
|
|
} else {
|
|
// Replace the original atomic instruction with the new one.
|
|
I.replaceAllUsesWith(Result);
|
|
}
|
|
}
|
|
|
|
// And delete the original.
|
|
I.eraseFromParent();
|
|
}
|
|
|
|
INITIALIZE_PASS_BEGIN(AMDGPUAtomicOptimizer, DEBUG_TYPE,
|
|
"AMDGPU atomic optimizations", false, false)
|
|
INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
|
|
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
|
|
INITIALIZE_PASS_END(AMDGPUAtomicOptimizer, DEBUG_TYPE,
|
|
"AMDGPU atomic optimizations", false, false)
|
|
|
|
FunctionPass *llvm::createAMDGPUAtomicOptimizerPass() {
|
|
return new AMDGPUAtomicOptimizer();
|
|
}
|