74b7abf154
Add a new `CreateIntrinsic` overload with no `Types`, useful for creating calls to non-overloaded intrinsics that don't need additional mangling.
371 lines
16 KiB
C++
371 lines
16 KiB
C++
//===AMDGPUAsanInstrumentation.cpp - ASAN related helper functions===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===-------------------------------------------------------------===//
|
|
|
|
#include "AMDGPUAsanInstrumentation.h"
|
|
|
|
#define DEBUG_TYPE "amdgpu-asan-instrumentation"
|
|
|
|
using namespace llvm;
|
|
|
|
namespace llvm {
|
|
namespace AMDGPU {
|
|
|
|
static uint64_t getRedzoneSizeForScale(int AsanScale) {
|
|
// Redzone used for stack and globals is at least 32 bytes.
|
|
// For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.
|
|
return std::max(32U, 1U << AsanScale);
|
|
}
|
|
|
|
static uint64_t getMinRedzoneSizeForGlobal(int AsanScale) {
|
|
return getRedzoneSizeForScale(AsanScale);
|
|
}
|
|
|
|
uint64_t getRedzoneSizeForGlobal(int AsanScale, uint64_t SizeInBytes) {
|
|
constexpr uint64_t kMaxRZ = 1 << 18;
|
|
const uint64_t MinRZ = getMinRedzoneSizeForGlobal(AsanScale);
|
|
|
|
uint64_t RZ = 0;
|
|
if (SizeInBytes <= MinRZ / 2) {
|
|
// Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is
|
|
// at least 32 bytes, optimize when SizeInBytes is less than or equal to
|
|
// half of MinRZ.
|
|
RZ = MinRZ - SizeInBytes;
|
|
} else {
|
|
// Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes.
|
|
RZ = std::clamp((SizeInBytes / MinRZ / 4) * MinRZ, MinRZ, kMaxRZ);
|
|
|
|
// Round up to multiple of MinRZ.
|
|
if (SizeInBytes % MinRZ)
|
|
RZ += MinRZ - (SizeInBytes % MinRZ);
|
|
}
|
|
|
|
assert((RZ + SizeInBytes) % MinRZ == 0);
|
|
|
|
return RZ;
|
|
}
|
|
|
|
static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) {
|
|
size_t Res = llvm::countr_zero(TypeSize / 8);
|
|
return Res;
|
|
}
|
|
|
|
static Instruction *genAMDGPUReportBlock(Module &M, IRBuilder<> &IRB,
|
|
Value *Cond, bool Recover) {
|
|
Value *ReportCond = Cond;
|
|
if (!Recover) {
|
|
auto *Ballot =
|
|
IRB.CreateIntrinsic(Intrinsic::amdgcn_ballot, IRB.getInt64Ty(), {Cond});
|
|
ReportCond = IRB.CreateIsNotNull(Ballot);
|
|
}
|
|
|
|
auto *Trm = SplitBlockAndInsertIfThen(
|
|
ReportCond, &*IRB.GetInsertPoint(), false,
|
|
MDBuilder(M.getContext()).createUnlikelyBranchWeights());
|
|
Trm->getParent()->setName("asan.report");
|
|
|
|
if (Recover)
|
|
return Trm;
|
|
|
|
Trm = SplitBlockAndInsertIfThen(Cond, Trm, false);
|
|
IRB.SetInsertPoint(Trm);
|
|
return IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {});
|
|
}
|
|
|
|
static Value *createSlowPathCmp(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
|
|
Value *AddrLong, Value *ShadowValue,
|
|
uint32_t TypeStoreSize, int AsanScale) {
|
|
uint64_t Granularity = static_cast<uint64_t>(1) << AsanScale;
|
|
// Addr & (Granularity - 1)
|
|
Value *LastAccessedByte =
|
|
IRB.CreateAnd(AddrLong, ConstantInt::get(IntptrTy, Granularity - 1));
|
|
// (Addr & (Granularity - 1)) + size - 1
|
|
if (TypeStoreSize / 8 > 1)
|
|
LastAccessedByte = IRB.CreateAdd(
|
|
LastAccessedByte, ConstantInt::get(IntptrTy, TypeStoreSize / 8 - 1));
|
|
// (uint8_t) ((Addr & (Granularity-1)) + size - 1)
|
|
LastAccessedByte =
|
|
IRB.CreateIntCast(LastAccessedByte, ShadowValue->getType(), false);
|
|
// ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue
|
|
return IRB.CreateICmpSGE(LastAccessedByte, ShadowValue);
|
|
}
|
|
|
|
static Instruction *generateCrashCode(Module &M, IRBuilder<> &IRB,
|
|
Type *IntptrTy, Instruction *InsertBefore,
|
|
Value *Addr, bool IsWrite,
|
|
size_t AccessSizeIndex,
|
|
Value *SizeArgument, bool Recover) {
|
|
IRB.SetInsertPoint(InsertBefore);
|
|
CallInst *Call = nullptr;
|
|
SmallString<128> kAsanReportErrorTemplate{"__asan_report_"};
|
|
SmallString<64> TypeStr{IsWrite ? "store" : "load"};
|
|
SmallString<64> EndingStr{Recover ? "_noabort" : ""};
|
|
|
|
SmallString<128> AsanErrorCallbackSizedString;
|
|
raw_svector_ostream AsanErrorCallbackSizedOS(AsanErrorCallbackSizedString);
|
|
AsanErrorCallbackSizedOS << kAsanReportErrorTemplate << TypeStr << "_n"
|
|
<< EndingStr;
|
|
|
|
SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
|
|
AttributeList AL2;
|
|
FunctionCallee AsanErrorCallbackSized = M.getOrInsertFunction(
|
|
AsanErrorCallbackSizedOS.str(),
|
|
FunctionType::get(IRB.getVoidTy(), Args2, false), AL2);
|
|
SmallVector<Type *, 2> Args1{1, IntptrTy};
|
|
AttributeList AL1;
|
|
|
|
SmallString<128> AsanErrorCallbackString;
|
|
raw_svector_ostream AsanErrorCallbackOS(AsanErrorCallbackString);
|
|
AsanErrorCallbackOS << kAsanReportErrorTemplate << TypeStr
|
|
<< (1ULL << AccessSizeIndex) << EndingStr;
|
|
|
|
FunctionCallee AsanErrorCallback = M.getOrInsertFunction(
|
|
AsanErrorCallbackOS.str(),
|
|
FunctionType::get(IRB.getVoidTy(), Args1, false), AL1);
|
|
if (SizeArgument) {
|
|
Call = IRB.CreateCall(AsanErrorCallbackSized, {Addr, SizeArgument});
|
|
} else {
|
|
Call = IRB.CreateCall(AsanErrorCallback, Addr);
|
|
}
|
|
|
|
Call->setCannotMerge();
|
|
return Call;
|
|
}
|
|
|
|
static Value *memToShadow(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
|
|
Value *Shadow, int AsanScale, uint32_t AsanOffset) {
|
|
// Shadow >> scale
|
|
Shadow = IRB.CreateLShr(Shadow, AsanScale);
|
|
if (AsanOffset == 0)
|
|
return Shadow;
|
|
// (Shadow >> scale) | offset
|
|
Value *ShadowBase = ConstantInt::get(IntptrTy, AsanOffset);
|
|
return IRB.CreateAdd(Shadow, ShadowBase);
|
|
}
|
|
|
|
static void instrumentAddressImpl(Module &M, IRBuilder<> &IRB,
|
|
Instruction *OrigIns,
|
|
Instruction *InsertBefore, Value *Addr,
|
|
Align Alignment, uint32_t TypeStoreSize,
|
|
bool IsWrite, Value *SizeArgument,
|
|
bool UseCalls, bool Recover, int AsanScale,
|
|
int AsanOffset) {
|
|
Type *AddrTy = Addr->getType();
|
|
Type *IntptrTy = M.getDataLayout().getIntPtrType(
|
|
M.getContext(), AddrTy->getPointerAddressSpace());
|
|
IRB.SetInsertPoint(InsertBefore);
|
|
size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeStoreSize);
|
|
Type *ShadowTy = IntegerType::get(M.getContext(),
|
|
std::max(8U, TypeStoreSize >> AsanScale));
|
|
Type *ShadowPtrTy = PointerType::get(M.getContext(), 0);
|
|
Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy);
|
|
Value *ShadowPtr =
|
|
memToShadow(M, IRB, IntptrTy, AddrLong, AsanScale, AsanOffset);
|
|
const uint64_t ShadowAlign =
|
|
std::max<uint64_t>(Alignment.value() >> AsanScale, 1);
|
|
Value *ShadowValue = IRB.CreateAlignedLoad(
|
|
ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy), Align(ShadowAlign));
|
|
Value *Cmp = IRB.CreateIsNotNull(ShadowValue);
|
|
auto *Cmp2 = createSlowPathCmp(M, IRB, IntptrTy, AddrLong, ShadowValue,
|
|
TypeStoreSize, AsanScale);
|
|
Cmp = IRB.CreateAnd(Cmp, Cmp2);
|
|
Instruction *CrashTerm = genAMDGPUReportBlock(M, IRB, Cmp, Recover);
|
|
Instruction *Crash =
|
|
generateCrashCode(M, IRB, IntptrTy, CrashTerm, AddrLong, IsWrite,
|
|
AccessSizeIndex, SizeArgument, Recover);
|
|
Crash->setDebugLoc(OrigIns->getDebugLoc());
|
|
}
|
|
|
|
void instrumentAddress(Module &M, IRBuilder<> &IRB, Instruction *OrigIns,
|
|
Instruction *InsertBefore, Value *Addr, Align Alignment,
|
|
TypeSize TypeStoreSize, bool IsWrite,
|
|
Value *SizeArgument, bool UseCalls, bool Recover,
|
|
int AsanScale, int AsanOffset) {
|
|
if (!TypeStoreSize.isScalable()) {
|
|
unsigned Granularity = 1 << AsanScale;
|
|
const auto FixedSize = TypeStoreSize.getFixedValue();
|
|
switch (FixedSize) {
|
|
case 8:
|
|
case 16:
|
|
case 32:
|
|
case 64:
|
|
case 128:
|
|
if (Alignment.value() >= Granularity ||
|
|
Alignment.value() >= FixedSize / 8)
|
|
return instrumentAddressImpl(
|
|
M, IRB, OrigIns, InsertBefore, Addr, Alignment, FixedSize, IsWrite,
|
|
SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
|
|
}
|
|
}
|
|
// Instrument unusual size or unusual alignment.
|
|
IRB.SetInsertPoint(InsertBefore);
|
|
Type *AddrTy = Addr->getType();
|
|
Type *IntptrTy = M.getDataLayout().getIntPtrType(AddrTy);
|
|
Value *NumBits = IRB.CreateTypeSize(IntptrTy, TypeStoreSize);
|
|
Value *Size = IRB.CreateLShr(NumBits, ConstantInt::get(IntptrTy, 3));
|
|
Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy);
|
|
Value *SizeMinusOne = IRB.CreateAdd(Size, ConstantInt::get(IntptrTy, -1));
|
|
Value *LastByte =
|
|
IRB.CreateIntToPtr(IRB.CreateAdd(AddrLong, SizeMinusOne), AddrTy);
|
|
instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, Addr, {}, 8, IsWrite,
|
|
SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
|
|
instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, LastByte, {}, 8, IsWrite,
|
|
SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
|
|
}
|
|
|
|
void getInterestingMemoryOperands(
|
|
Module &M, Instruction *I,
|
|
SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
|
|
const DataLayout &DL = M.getDataLayout();
|
|
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
|
|
Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
|
|
LI->getType(), LI->getAlign());
|
|
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
|
|
Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
|
|
SI->getValueOperand()->getType(), SI->getAlign());
|
|
} else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
|
|
Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
|
|
RMW->getValOperand()->getType(), std::nullopt);
|
|
} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
|
|
Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
|
|
XCHG->getCompareOperand()->getType(),
|
|
std::nullopt);
|
|
} else if (auto *CI = dyn_cast<CallInst>(I)) {
|
|
switch (CI->getIntrinsicID()) {
|
|
case Intrinsic::masked_load:
|
|
case Intrinsic::masked_store:
|
|
case Intrinsic::masked_gather:
|
|
case Intrinsic::masked_scatter: {
|
|
bool IsWrite = CI->getType()->isVoidTy();
|
|
// Masked store has an initial operand for the value.
|
|
unsigned OpOffset = IsWrite ? 1 : 0;
|
|
Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
|
|
MaybeAlign Alignment = Align(1);
|
|
// Otherwise no alignment guarantees. We probably got Undef.
|
|
if (auto *Op = dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
|
|
Alignment = Op->getMaybeAlignValue();
|
|
Value *Mask = CI->getOperand(2 + OpOffset);
|
|
Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, Mask);
|
|
break;
|
|
}
|
|
case Intrinsic::masked_expandload:
|
|
case Intrinsic::masked_compressstore: {
|
|
bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;
|
|
unsigned OpOffset = IsWrite ? 1 : 0;
|
|
auto *BasePtr = CI->getOperand(OpOffset);
|
|
MaybeAlign Alignment = BasePtr->getPointerAlignment(DL);
|
|
Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
|
|
IRBuilder<> IB(I);
|
|
Value *Mask = CI->getOperand(1 + OpOffset);
|
|
Type *IntptrTy = M.getDataLayout().getIntPtrType(
|
|
M.getContext(), BasePtr->getType()->getPointerAddressSpace());
|
|
// Use the popcount of Mask as the effective vector length.
|
|
Type *ExtTy = VectorType::get(IntptrTy, cast<VectorType>(Ty));
|
|
Value *ExtMask = IB.CreateZExt(Mask, ExtTy);
|
|
Value *EVL = IB.CreateAddReduce(ExtMask);
|
|
Value *TrueMask = ConstantInt::get(Mask->getType(), 1);
|
|
Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, TrueMask,
|
|
EVL);
|
|
break;
|
|
}
|
|
case Intrinsic::vp_load:
|
|
case Intrinsic::vp_store:
|
|
case Intrinsic::experimental_vp_strided_load:
|
|
case Intrinsic::experimental_vp_strided_store: {
|
|
auto *VPI = cast<VPIntrinsic>(CI);
|
|
unsigned IID = CI->getIntrinsicID();
|
|
bool IsWrite = CI->getType()->isVoidTy();
|
|
unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
|
|
Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
|
|
MaybeAlign Alignment = VPI->getOperand(PtrOpNo)->getPointerAlignment(DL);
|
|
Value *Stride = nullptr;
|
|
if (IID == Intrinsic::experimental_vp_strided_store ||
|
|
IID == Intrinsic::experimental_vp_strided_load) {
|
|
Stride = VPI->getOperand(PtrOpNo + 1);
|
|
// Use the pointer alignment as the element alignment if the stride is a
|
|
// mutiple of the pointer alignment. Otherwise, the element alignment
|
|
// should be Align(1).
|
|
unsigned PointerAlign = Alignment.valueOrOne().value();
|
|
if (!isa<ConstantInt>(Stride) ||
|
|
cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)
|
|
Alignment = Align(1);
|
|
}
|
|
Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,
|
|
VPI->getMaskParam(), VPI->getVectorLengthParam(),
|
|
Stride);
|
|
break;
|
|
}
|
|
case Intrinsic::vp_gather:
|
|
case Intrinsic::vp_scatter: {
|
|
auto *VPI = cast<VPIntrinsic>(CI);
|
|
unsigned IID = CI->getIntrinsicID();
|
|
bool IsWrite = IID == Intrinsic::vp_scatter;
|
|
unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
|
|
Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
|
|
MaybeAlign Alignment = VPI->getPointerAlignment();
|
|
Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,
|
|
VPI->getMaskParam(),
|
|
VPI->getVectorLengthParam());
|
|
break;
|
|
}
|
|
case Intrinsic::amdgcn_raw_buffer_load:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_load:
|
|
case Intrinsic::amdgcn_raw_buffer_load_format:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
|
|
case Intrinsic::amdgcn_raw_tbuffer_load:
|
|
case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
|
|
case Intrinsic::amdgcn_struct_buffer_load:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_load:
|
|
case Intrinsic::amdgcn_struct_buffer_load_format:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
|
|
case Intrinsic::amdgcn_struct_tbuffer_load:
|
|
case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
|
|
case Intrinsic::amdgcn_s_buffer_load:
|
|
case Intrinsic::amdgcn_global_load_tr_b64:
|
|
case Intrinsic::amdgcn_global_load_tr_b128: {
|
|
unsigned PtrOpNo = 0;
|
|
bool IsWrite = false;
|
|
Type *Ty = CI->getType();
|
|
Value *Ptr = CI->getArgOperand(PtrOpNo);
|
|
MaybeAlign Alignment = Ptr->getPointerAlignment(DL);
|
|
Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment);
|
|
break;
|
|
}
|
|
case Intrinsic::amdgcn_raw_tbuffer_store:
|
|
case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
|
|
case Intrinsic::amdgcn_raw_buffer_store:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_store:
|
|
case Intrinsic::amdgcn_raw_buffer_store_format:
|
|
case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
|
|
case Intrinsic::amdgcn_struct_buffer_store:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_store:
|
|
case Intrinsic::amdgcn_struct_buffer_store_format:
|
|
case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
|
|
case Intrinsic::amdgcn_struct_tbuffer_store:
|
|
case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
|
|
unsigned PtrOpNo = 1;
|
|
bool IsWrite = true;
|
|
Value *Ptr = CI->getArgOperand(PtrOpNo);
|
|
Type *Ty = Ptr->getType();
|
|
MaybeAlign Alignment = Ptr->getPointerAlignment(DL);
|
|
Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment);
|
|
break;
|
|
}
|
|
default:
|
|
for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
|
|
if (Type *Ty = CI->getParamByRefType(ArgNo)) {
|
|
Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
|
|
} else if (Type *Ty = CI->getParamByValType(ArgNo)) {
|
|
Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} // end namespace AMDGPU
|
|
} // end namespace llvm
|