8baa212b7a
We can end up with loads of single element vectors when we have scalar values, because the vectorizer may introduce these to use ops like shufflevector in some cases. Make sure we're maintaining the correct type when translating these into resource load operations. Fixes #136409.
308 lines
11 KiB
C++
308 lines
11 KiB
C++
//===- DXILResourceAccess.cpp - Resource access via load/store ------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "DXILResourceAccess.h"
|
|
#include "DirectX.h"
|
|
#include "llvm/Analysis/DXILResource.h"
|
|
#include "llvm/IR/Dominators.h"
|
|
#include "llvm/IR/IRBuilder.h"
|
|
#include "llvm/IR/Instructions.h"
|
|
#include "llvm/IR/IntrinsicInst.h"
|
|
#include "llvm/IR/Intrinsics.h"
|
|
#include "llvm/IR/IntrinsicsDirectX.h"
|
|
#include "llvm/InitializePasses.h"
|
|
|
|
#define DEBUG_TYPE "dxil-resource-access"
|
|
|
|
using namespace llvm;
|
|
|
|
static Value *calculateGEPOffset(GetElementPtrInst *GEP, Value *PrevOffset,
|
|
dxil::ResourceTypeInfo &RTI) {
|
|
assert(!PrevOffset && "Non-constant GEP chains not handled yet");
|
|
|
|
const DataLayout &DL = GEP->getDataLayout();
|
|
|
|
uint64_t ScalarSize = 1;
|
|
if (RTI.isTyped()) {
|
|
Type *ContainedType = RTI.getHandleTy()->getTypeParameter(0);
|
|
// We need the size of an element in bytes so that we can calculate the
|
|
// offset in elements given a total offset in bytes.
|
|
Type *ScalarType = ContainedType->getScalarType();
|
|
ScalarSize = DL.getTypeSizeInBits(ScalarType) / 8;
|
|
}
|
|
|
|
APInt ConstantOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
|
|
if (GEP->accumulateConstantOffset(DL, ConstantOffset)) {
|
|
APInt Scaled = ConstantOffset.udiv(ScalarSize);
|
|
return ConstantInt::get(Type::getInt32Ty(GEP->getContext()), Scaled);
|
|
}
|
|
|
|
auto IndexIt = GEP->idx_begin();
|
|
assert(cast<ConstantInt>(IndexIt)->getZExtValue() == 0 &&
|
|
"GEP is not indexing through pointer");
|
|
++IndexIt;
|
|
Value *Offset = *IndexIt;
|
|
assert(++IndexIt == GEP->idx_end() && "Too many indices in GEP");
|
|
return Offset;
|
|
}
|
|
|
|
static void createTypedBufferStore(IntrinsicInst *II, StoreInst *SI,
|
|
Value *Offset, dxil::ResourceTypeInfo &RTI) {
|
|
IRBuilder<> Builder(SI);
|
|
Type *ContainedType = RTI.getHandleTy()->getTypeParameter(0);
|
|
Type *LoadType = StructType::get(ContainedType, Builder.getInt1Ty());
|
|
|
|
Value *V = SI->getValueOperand();
|
|
if (V->getType() == ContainedType) {
|
|
// V is already the right type.
|
|
assert(!Offset && "store of whole element has offset?");
|
|
} else if (V->getType() == ContainedType->getScalarType()) {
|
|
// We're storing a scalar, so we need to load the current value and only
|
|
// replace the relevant part.
|
|
auto *Load = Builder.CreateIntrinsic(
|
|
LoadType, Intrinsic::dx_resource_load_typedbuffer,
|
|
{II->getOperand(0), II->getOperand(1)});
|
|
auto *Struct = Builder.CreateExtractValue(Load, {0});
|
|
|
|
// If we have an offset from seeing a GEP earlier, use that. Otherwise, 0.
|
|
if (!Offset)
|
|
Offset = ConstantInt::get(Builder.getInt32Ty(), 0);
|
|
V = Builder.CreateInsertElement(Struct, V, Offset);
|
|
} else {
|
|
llvm_unreachable("Store to typed resource has invalid type");
|
|
}
|
|
|
|
auto *Inst = Builder.CreateIntrinsic(
|
|
Builder.getVoidTy(), Intrinsic::dx_resource_store_typedbuffer,
|
|
{II->getOperand(0), II->getOperand(1), V});
|
|
SI->replaceAllUsesWith(Inst);
|
|
}
|
|
|
|
static void createRawStore(IntrinsicInst *II, StoreInst *SI, Value *Offset) {
|
|
IRBuilder<> Builder(SI);
|
|
|
|
if (!Offset)
|
|
Offset = ConstantInt::get(Builder.getInt32Ty(), 0);
|
|
Value *V = SI->getValueOperand();
|
|
// TODO: break up larger types
|
|
auto *Inst = Builder.CreateIntrinsic(
|
|
Builder.getVoidTy(), Intrinsic::dx_resource_store_rawbuffer,
|
|
{II->getOperand(0), II->getOperand(1), Offset, V});
|
|
SI->replaceAllUsesWith(Inst);
|
|
}
|
|
|
|
static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI,
|
|
Value *Offset, dxil::ResourceTypeInfo &RTI) {
|
|
switch (RTI.getResourceKind()) {
|
|
case dxil::ResourceKind::TypedBuffer:
|
|
return createTypedBufferStore(II, SI, Offset, RTI);
|
|
case dxil::ResourceKind::RawBuffer:
|
|
case dxil::ResourceKind::StructuredBuffer:
|
|
return createRawStore(II, SI, Offset);
|
|
case dxil::ResourceKind::Texture1D:
|
|
case dxil::ResourceKind::Texture2D:
|
|
case dxil::ResourceKind::Texture2DMS:
|
|
case dxil::ResourceKind::Texture3D:
|
|
case dxil::ResourceKind::TextureCube:
|
|
case dxil::ResourceKind::Texture1DArray:
|
|
case dxil::ResourceKind::Texture2DArray:
|
|
case dxil::ResourceKind::Texture2DMSArray:
|
|
case dxil::ResourceKind::TextureCubeArray:
|
|
case dxil::ResourceKind::FeedbackTexture2D:
|
|
case dxil::ResourceKind::FeedbackTexture2DArray:
|
|
report_fatal_error("DXIL Load not implemented yet",
|
|
/*gen_crash_diag=*/false);
|
|
return;
|
|
case dxil::ResourceKind::CBuffer:
|
|
case dxil::ResourceKind::Sampler:
|
|
case dxil::ResourceKind::TBuffer:
|
|
case dxil::ResourceKind::RTAccelerationStructure:
|
|
case dxil::ResourceKind::Invalid:
|
|
case dxil::ResourceKind::NumEntries:
|
|
llvm_unreachable("Invalid resource kind for store");
|
|
}
|
|
llvm_unreachable("Unhandled case in switch");
|
|
}
|
|
|
|
static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI,
|
|
Value *Offset, dxil::ResourceTypeInfo &RTI) {
|
|
IRBuilder<> Builder(LI);
|
|
Type *ContainedType = RTI.getHandleTy()->getTypeParameter(0);
|
|
Type *LoadType = StructType::get(ContainedType, Builder.getInt1Ty());
|
|
|
|
Value *V =
|
|
Builder.CreateIntrinsic(LoadType, Intrinsic::dx_resource_load_typedbuffer,
|
|
{II->getOperand(0), II->getOperand(1)});
|
|
V = Builder.CreateExtractValue(V, {0});
|
|
|
|
if (Offset)
|
|
V = Builder.CreateExtractElement(V, Offset);
|
|
|
|
// If we loaded a <1 x ...> instead of a scalar (presumably to feed a
|
|
// shufflevector), then make sure we're maintaining the resulting type.
|
|
if (auto *VT = dyn_cast<FixedVectorType>(LI->getType()))
|
|
if (VT->getNumElements() == 1 && !isa<FixedVectorType>(V->getType()))
|
|
V = Builder.CreateInsertElement(PoisonValue::get(VT), V,
|
|
Builder.getInt32(0));
|
|
|
|
LI->replaceAllUsesWith(V);
|
|
}
|
|
|
|
static void createRawLoad(IntrinsicInst *II, LoadInst *LI, Value *Offset) {
|
|
IRBuilder<> Builder(LI);
|
|
// TODO: break up larger types
|
|
Type *LoadType = StructType::get(LI->getType(), Builder.getInt1Ty());
|
|
if (!Offset)
|
|
Offset = ConstantInt::get(Builder.getInt32Ty(), 0);
|
|
Value *V =
|
|
Builder.CreateIntrinsic(LoadType, Intrinsic::dx_resource_load_rawbuffer,
|
|
{II->getOperand(0), II->getOperand(1), Offset});
|
|
V = Builder.CreateExtractValue(V, {0});
|
|
|
|
LI->replaceAllUsesWith(V);
|
|
}
|
|
|
|
static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI, Value *Offset,
|
|
dxil::ResourceTypeInfo &RTI) {
|
|
switch (RTI.getResourceKind()) {
|
|
case dxil::ResourceKind::TypedBuffer:
|
|
return createTypedBufferLoad(II, LI, Offset, RTI);
|
|
case dxil::ResourceKind::RawBuffer:
|
|
case dxil::ResourceKind::StructuredBuffer:
|
|
return createRawLoad(II, LI, Offset);
|
|
case dxil::ResourceKind::Texture1D:
|
|
case dxil::ResourceKind::Texture2D:
|
|
case dxil::ResourceKind::Texture2DMS:
|
|
case dxil::ResourceKind::Texture3D:
|
|
case dxil::ResourceKind::TextureCube:
|
|
case dxil::ResourceKind::Texture1DArray:
|
|
case dxil::ResourceKind::Texture2DArray:
|
|
case dxil::ResourceKind::Texture2DMSArray:
|
|
case dxil::ResourceKind::TextureCubeArray:
|
|
case dxil::ResourceKind::FeedbackTexture2D:
|
|
case dxil::ResourceKind::FeedbackTexture2DArray:
|
|
case dxil::ResourceKind::CBuffer:
|
|
case dxil::ResourceKind::TBuffer:
|
|
// TODO: handle these
|
|
return;
|
|
case dxil::ResourceKind::Sampler:
|
|
case dxil::ResourceKind::RTAccelerationStructure:
|
|
case dxil::ResourceKind::Invalid:
|
|
case dxil::ResourceKind::NumEntries:
|
|
llvm_unreachable("Invalid resource kind for load");
|
|
}
|
|
llvm_unreachable("Unhandled case in switch");
|
|
}
|
|
|
|
static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {
|
|
// Process users keeping track of indexing accumulated from GEPs.
|
|
struct AccessAndOffset {
|
|
User *Access;
|
|
Value *Offset;
|
|
};
|
|
SmallVector<AccessAndOffset> Worklist;
|
|
for (User *U : II->users())
|
|
Worklist.push_back({U, nullptr});
|
|
|
|
SmallVector<Instruction *> DeadInsts;
|
|
while (!Worklist.empty()) {
|
|
AccessAndOffset Current = Worklist.back();
|
|
Worklist.pop_back();
|
|
|
|
if (auto *GEP = dyn_cast<GetElementPtrInst>(Current.Access)) {
|
|
IRBuilder<> Builder(GEP);
|
|
|
|
Value *Offset = calculateGEPOffset(GEP, Current.Offset, RTI);
|
|
for (User *U : GEP->users())
|
|
Worklist.push_back({U, Offset});
|
|
DeadInsts.push_back(GEP);
|
|
|
|
} else if (auto *SI = dyn_cast<StoreInst>(Current.Access)) {
|
|
assert(SI->getValueOperand() != II && "Pointer escaped!");
|
|
createStoreIntrinsic(II, SI, Current.Offset, RTI);
|
|
DeadInsts.push_back(SI);
|
|
|
|
} else if (auto *LI = dyn_cast<LoadInst>(Current.Access)) {
|
|
createLoadIntrinsic(II, LI, Current.Offset, RTI);
|
|
DeadInsts.push_back(LI);
|
|
|
|
} else
|
|
llvm_unreachable("Unhandled instruction - pointer escaped?");
|
|
}
|
|
|
|
// Traverse the now-dead instructions in RPO and remove them.
|
|
for (Instruction *Dead : llvm::reverse(DeadInsts))
|
|
Dead->eraseFromParent();
|
|
II->eraseFromParent();
|
|
}
|
|
|
|
static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) {
|
|
bool Changed = false;
|
|
SmallVector<std::pair<IntrinsicInst *, dxil::ResourceTypeInfo>> Resources;
|
|
for (BasicBlock &BB : F)
|
|
for (Instruction &I : BB)
|
|
if (auto *II = dyn_cast<IntrinsicInst>(&I))
|
|
if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer) {
|
|
auto *HandleTy = cast<TargetExtType>(II->getArgOperand(0)->getType());
|
|
Resources.emplace_back(II, DRTM[HandleTy]);
|
|
}
|
|
|
|
for (auto &[II, RI] : Resources)
|
|
replaceAccess(II, RI);
|
|
|
|
return Changed;
|
|
}
|
|
|
|
PreservedAnalyses DXILResourceAccess::run(Function &F,
|
|
FunctionAnalysisManager &FAM) {
|
|
auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
|
|
DXILResourceTypeMap *DRTM =
|
|
MAMProxy.getCachedResult<DXILResourceTypeAnalysis>(*F.getParent());
|
|
assert(DRTM && "DXILResourceTypeAnalysis must be available");
|
|
|
|
bool MadeChanges = transformResourcePointers(F, *DRTM);
|
|
if (!MadeChanges)
|
|
return PreservedAnalyses::all();
|
|
|
|
PreservedAnalyses PA;
|
|
PA.preserve<DXILResourceTypeAnalysis>();
|
|
PA.preserve<DominatorTreeAnalysis>();
|
|
return PA;
|
|
}
|
|
|
|
namespace {
|
|
class DXILResourceAccessLegacy : public FunctionPass {
|
|
public:
|
|
bool runOnFunction(Function &F) override {
|
|
DXILResourceTypeMap &DRTM =
|
|
getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
|
|
|
|
return transformResourcePointers(F, DRTM);
|
|
}
|
|
StringRef getPassName() const override { return "DXIL Resource Access"; }
|
|
DXILResourceAccessLegacy() : FunctionPass(ID) {}
|
|
|
|
static char ID; // Pass identification.
|
|
void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
|
|
AU.addRequired<DXILResourceTypeWrapperPass>();
|
|
AU.addPreserved<DominatorTreeWrapperPass>();
|
|
}
|
|
};
|
|
char DXILResourceAccessLegacy::ID = 0;
|
|
} // end anonymous namespace
|
|
|
|
INITIALIZE_PASS_BEGIN(DXILResourceAccessLegacy, DEBUG_TYPE,
|
|
"DXIL Resource Access", false, false)
|
|
INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
|
|
INITIALIZE_PASS_END(DXILResourceAccessLegacy, DEBUG_TYPE,
|
|
"DXIL Resource Access", false, false)
|
|
|
|
FunctionPass *llvm::createDXILResourceAccessLegacyPass() {
|
|
return new DXILResourceAccessLegacy();
|
|
}
|