5c84054223
Summary: The standard GPU compilation process embeds each intermediate object file into the host file at the `.llvm.offloading` section so it can be linked later. We also use a special section called something like `omp_offloading_entries` to store all the globals that need to be registered by the runtime. The linker-wrapper's job is to link the embedded device code stored at this section and then emit code to register the linked image and the kernels and globals in the offloading entry section. One downside to RDC linking is that it can become quite big for very large projects that wish to make use of static linking. This patch changes the support for relocatable linking via `-r` to support a kind of "partial" RDC compilation for offloading languages. This primarily requires manually editing the embedded data in the output object file for the relocatable link. We need to rename the output section to make it distinct from the input sections that will be merged. We then delete the old embedded object code so it won't be linked further. We then need to rename the old offloading section so that it is private to the module. A runtime solution could also be done to defer entries that don't belong to the given GPU executable, but this is easier. Note that this does not work with COFF linking, only the ELF method for handling offloading entries, that could be made to work similarly. Given this support, the following compilation path should produce two distinct images for OpenMP offloading. ``` $ clang foo.c -fopenmp --offload-arch=native -c $ clang foo.c -lomptarget.devicertl --offload-link -r -o merged.o $ clang main.c merged.o -fopenmp --offload-arch=native $ ./a.out ``` Or similarly for HIP to effectively perform non-RDC mode compilation for a subset of files. ``` $ clang -x hip foo.c --offload-arch=native --offload-new-driver -fgpu-rdc -c $ clang -x hip foo.c -lomptarget.devicertl --offload-link -r -o merged.o $ clang -x hip main.c merged.o --offload-arch=native --offload-new-driver -fgpu-rdc $ ./a.out ``` One question is whether or not this should be the default behavior of `-r` when run through the linker-wrapper or a special option. Standard `-r` behavior is still possible if used without invoking the linker-wrapper and it guaranteed to be correct.
624 lines
27 KiB
C++
624 lines
27 KiB
C++
//===- OffloadWrapper.cpp ---------------------------------------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/Frontend/Offloading/OffloadWrapper.h"
|
|
#include "llvm/ADT/ArrayRef.h"
|
|
#include "llvm/BinaryFormat/Magic.h"
|
|
#include "llvm/Frontend/Offloading/Utility.h"
|
|
#include "llvm/IR/Constants.h"
|
|
#include "llvm/IR/GlobalVariable.h"
|
|
#include "llvm/IR/IRBuilder.h"
|
|
#include "llvm/IR/LLVMContext.h"
|
|
#include "llvm/IR/Module.h"
|
|
#include "llvm/Object/OffloadBinary.h"
|
|
#include "llvm/Support/Error.h"
|
|
#include "llvm/TargetParser/Triple.h"
|
|
#include "llvm/Transforms/Utils/ModuleUtils.h"
|
|
|
|
using namespace llvm;
|
|
using namespace llvm::offloading;
|
|
|
|
namespace {
|
|
/// Magic number that begins the section containing the CUDA fatbinary.
|
|
constexpr unsigned CudaFatMagic = 0x466243b1;
|
|
constexpr unsigned HIPFatMagic = 0x48495046;
|
|
|
|
IntegerType *getSizeTTy(Module &M) {
|
|
return M.getDataLayout().getIntPtrType(M.getContext());
|
|
}
|
|
|
|
// struct __tgt_device_image {
|
|
// void *ImageStart;
|
|
// void *ImageEnd;
|
|
// __tgt_offload_entry *EntriesBegin;
|
|
// __tgt_offload_entry *EntriesEnd;
|
|
// };
|
|
StructType *getDeviceImageTy(Module &M) {
|
|
LLVMContext &C = M.getContext();
|
|
StructType *ImageTy = StructType::getTypeByName(C, "__tgt_device_image");
|
|
if (!ImageTy)
|
|
ImageTy =
|
|
StructType::create("__tgt_device_image", PointerType::getUnqual(C),
|
|
PointerType::getUnqual(C), PointerType::getUnqual(C),
|
|
PointerType::getUnqual(C));
|
|
return ImageTy;
|
|
}
|
|
|
|
PointerType *getDeviceImagePtrTy(Module &M) {
|
|
return PointerType::getUnqual(getDeviceImageTy(M));
|
|
}
|
|
|
|
// struct __tgt_bin_desc {
|
|
// int32_t NumDeviceImages;
|
|
// __tgt_device_image *DeviceImages;
|
|
// __tgt_offload_entry *HostEntriesBegin;
|
|
// __tgt_offload_entry *HostEntriesEnd;
|
|
// };
|
|
StructType *getBinDescTy(Module &M) {
|
|
LLVMContext &C = M.getContext();
|
|
StructType *DescTy = StructType::getTypeByName(C, "__tgt_bin_desc");
|
|
if (!DescTy)
|
|
DescTy = StructType::create(
|
|
"__tgt_bin_desc", Type::getInt32Ty(C), getDeviceImagePtrTy(M),
|
|
PointerType::getUnqual(C), PointerType::getUnqual(C));
|
|
return DescTy;
|
|
}
|
|
|
|
PointerType *getBinDescPtrTy(Module &M) {
|
|
return PointerType::getUnqual(getBinDescTy(M));
|
|
}
|
|
|
|
/// Creates binary descriptor for the given device images. Binary descriptor
|
|
/// is an object that is passed to the offloading runtime at program startup
|
|
/// and it describes all device images available in the executable or shared
|
|
/// library. It is defined as follows
|
|
///
|
|
/// __attribute__((visibility("hidden")))
|
|
/// extern __tgt_offload_entry *__start_omp_offloading_entries;
|
|
/// __attribute__((visibility("hidden")))
|
|
/// extern __tgt_offload_entry *__stop_omp_offloading_entries;
|
|
///
|
|
/// static const char Image0[] = { <Bufs.front() contents> };
|
|
/// ...
|
|
/// static const char ImageN[] = { <Bufs.back() contents> };
|
|
///
|
|
/// static const __tgt_device_image Images[] = {
|
|
/// {
|
|
/// Image0, /*ImageStart*/
|
|
/// Image0 + sizeof(Image0), /*ImageEnd*/
|
|
/// __start_omp_offloading_entries, /*EntriesBegin*/
|
|
/// __stop_omp_offloading_entries /*EntriesEnd*/
|
|
/// },
|
|
/// ...
|
|
/// {
|
|
/// ImageN, /*ImageStart*/
|
|
/// ImageN + sizeof(ImageN), /*ImageEnd*/
|
|
/// __start_omp_offloading_entries, /*EntriesBegin*/
|
|
/// __stop_omp_offloading_entries /*EntriesEnd*/
|
|
/// }
|
|
/// };
|
|
///
|
|
/// static const __tgt_bin_desc BinDesc = {
|
|
/// sizeof(Images) / sizeof(Images[0]), /*NumDeviceImages*/
|
|
/// Images, /*DeviceImages*/
|
|
/// __start_omp_offloading_entries, /*HostEntriesBegin*/
|
|
/// __stop_omp_offloading_entries /*HostEntriesEnd*/
|
|
/// };
|
|
///
|
|
/// Global variable that represents BinDesc is returned.
|
|
GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,
|
|
EntryArrayTy EntryArray, StringRef Suffix,
|
|
bool Relocatable) {
|
|
LLVMContext &C = M.getContext();
|
|
auto [EntriesB, EntriesE] = EntryArray;
|
|
|
|
auto *Zero = ConstantInt::get(getSizeTTy(M), 0u);
|
|
Constant *ZeroZero[] = {Zero, Zero};
|
|
|
|
// Create initializer for the images array.
|
|
SmallVector<Constant *, 4u> ImagesInits;
|
|
ImagesInits.reserve(Bufs.size());
|
|
for (ArrayRef<char> Buf : Bufs) {
|
|
// We embed the full offloading entry so the binary utilities can parse it.
|
|
auto *Data = ConstantDataArray::get(C, Buf);
|
|
auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true,
|
|
GlobalVariable::InternalLinkage, Data,
|
|
".omp_offloading.device_image" + Suffix);
|
|
Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
|
|
Image->setSection(Relocatable ? ".llvm.offloading.relocatable"
|
|
: ".llvm.offloading");
|
|
Image->setAlignment(Align(object::OffloadBinary::getAlignment()));
|
|
|
|
StringRef Binary(Buf.data(), Buf.size());
|
|
assert(identify_magic(Binary) == file_magic::offload_binary &&
|
|
"Invalid binary format");
|
|
|
|
// The device image struct contains the pointer to the beginning and end of
|
|
// the image stored inside of the offload binary. There should only be one
|
|
// of these for each buffer so we parse it out manually.
|
|
const auto *Header =
|
|
reinterpret_cast<const object::OffloadBinary::Header *>(
|
|
Binary.bytes_begin());
|
|
const auto *Entry = reinterpret_cast<const object::OffloadBinary::Entry *>(
|
|
Binary.bytes_begin() + Header->EntryOffset);
|
|
|
|
auto *Begin = ConstantInt::get(getSizeTTy(M), Entry->ImageOffset);
|
|
auto *Size =
|
|
ConstantInt::get(getSizeTTy(M), Entry->ImageOffset + Entry->ImageSize);
|
|
Constant *ZeroBegin[] = {Zero, Begin};
|
|
Constant *ZeroSize[] = {Zero, Size};
|
|
|
|
auto *ImageB =
|
|
ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroBegin);
|
|
auto *ImageE =
|
|
ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroSize);
|
|
|
|
ImagesInits.push_back(ConstantStruct::get(getDeviceImageTy(M), ImageB,
|
|
ImageE, EntriesB, EntriesE));
|
|
}
|
|
|
|
// Then create images array.
|
|
auto *ImagesData = ConstantArray::get(
|
|
ArrayType::get(getDeviceImageTy(M), ImagesInits.size()), ImagesInits);
|
|
|
|
auto *Images =
|
|
new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,
|
|
GlobalValue::InternalLinkage, ImagesData,
|
|
".omp_offloading.device_images" + Suffix);
|
|
Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
|
|
|
|
auto *ImagesB =
|
|
ConstantExpr::getGetElementPtr(Images->getValueType(), Images, ZeroZero);
|
|
|
|
// And finally create the binary descriptor object.
|
|
auto *DescInit = ConstantStruct::get(
|
|
getBinDescTy(M),
|
|
ConstantInt::get(Type::getInt32Ty(C), ImagesInits.size()), ImagesB,
|
|
EntriesB, EntriesE);
|
|
|
|
return new GlobalVariable(M, DescInit->getType(), /*isConstant*/ true,
|
|
GlobalValue::InternalLinkage, DescInit,
|
|
".omp_offloading.descriptor" + Suffix);
|
|
}
|
|
|
|
void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
|
|
StringRef Suffix) {
|
|
LLVMContext &C = M.getContext();
|
|
auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
|
|
auto *Func = Function::Create(FuncTy, GlobalValue::InternalLinkage,
|
|
".omp_offloading.descriptor_reg" + Suffix, &M);
|
|
Func->setSection(".text.startup");
|
|
|
|
// Get __tgt_register_lib function declaration.
|
|
auto *RegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M),
|
|
/*isVarArg*/ false);
|
|
FunctionCallee RegFuncC =
|
|
M.getOrInsertFunction("__tgt_register_lib", RegFuncTy);
|
|
|
|
// Construct function body
|
|
IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
|
|
Builder.CreateCall(RegFuncC, BinDesc);
|
|
Builder.CreateRetVoid();
|
|
|
|
// Add this function to constructors.
|
|
// Set priority to 1 so that __tgt_register_lib is executed AFTER
|
|
// __tgt_register_requires (we want to know what requirements have been
|
|
// asked for before we load a libomptarget plugin so that by the time the
|
|
// plugin is loaded it can report how many devices there are which can
|
|
// satisfy these requirements).
|
|
appendToGlobalCtors(M, Func, /*Priority*/ 1);
|
|
}
|
|
|
|
void createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
|
|
StringRef Suffix) {
|
|
LLVMContext &C = M.getContext();
|
|
auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
|
|
auto *Func =
|
|
Function::Create(FuncTy, GlobalValue::InternalLinkage,
|
|
".omp_offloading.descriptor_unreg" + Suffix, &M);
|
|
Func->setSection(".text.startup");
|
|
|
|
// Get __tgt_unregister_lib function declaration.
|
|
auto *UnRegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M),
|
|
/*isVarArg*/ false);
|
|
FunctionCallee UnRegFuncC =
|
|
M.getOrInsertFunction("__tgt_unregister_lib", UnRegFuncTy);
|
|
|
|
// Construct function body
|
|
IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
|
|
Builder.CreateCall(UnRegFuncC, BinDesc);
|
|
Builder.CreateRetVoid();
|
|
|
|
// Add this function to global destructors.
|
|
// Match priority of __tgt_register_lib
|
|
appendToGlobalDtors(M, Func, /*Priority*/ 1);
|
|
}
|
|
|
|
// struct fatbin_wrapper {
|
|
// int32_t magic;
|
|
// int32_t version;
|
|
// void *image;
|
|
// void *reserved;
|
|
//};
|
|
StructType *getFatbinWrapperTy(Module &M) {
|
|
LLVMContext &C = M.getContext();
|
|
StructType *FatbinTy = StructType::getTypeByName(C, "fatbin_wrapper");
|
|
if (!FatbinTy)
|
|
FatbinTy = StructType::create(
|
|
"fatbin_wrapper", Type::getInt32Ty(C), Type::getInt32Ty(C),
|
|
PointerType::getUnqual(C), PointerType::getUnqual(C));
|
|
return FatbinTy;
|
|
}
|
|
|
|
/// Embed the image \p Image into the module \p M so it can be found by the
|
|
/// runtime.
|
|
GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP,
|
|
StringRef Suffix) {
|
|
LLVMContext &C = M.getContext();
|
|
llvm::Type *Int8PtrTy = PointerType::getUnqual(C);
|
|
llvm::Triple Triple = llvm::Triple(M.getTargetTriple());
|
|
|
|
// Create the global string containing the fatbinary.
|
|
StringRef FatbinConstantSection =
|
|
IsHIP ? ".hip_fatbin"
|
|
: (Triple.isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin");
|
|
auto *Data = ConstantDataArray::get(C, Image);
|
|
auto *Fatbin = new GlobalVariable(M, Data->getType(), /*isConstant*/ true,
|
|
GlobalVariable::InternalLinkage, Data,
|
|
".fatbin_image" + Suffix);
|
|
Fatbin->setSection(FatbinConstantSection);
|
|
|
|
// Create the fatbinary wrapper
|
|
StringRef FatbinWrapperSection = IsHIP ? ".hipFatBinSegment"
|
|
: Triple.isMacOSX() ? "__NV_CUDA,__fatbin"
|
|
: ".nvFatBinSegment";
|
|
Constant *FatbinWrapper[] = {
|
|
ConstantInt::get(Type::getInt32Ty(C), IsHIP ? HIPFatMagic : CudaFatMagic),
|
|
ConstantInt::get(Type::getInt32Ty(C), 1),
|
|
ConstantExpr::getPointerBitCastOrAddrSpaceCast(Fatbin, Int8PtrTy),
|
|
ConstantPointerNull::get(PointerType::getUnqual(C))};
|
|
|
|
Constant *FatbinInitializer =
|
|
ConstantStruct::get(getFatbinWrapperTy(M), FatbinWrapper);
|
|
|
|
auto *FatbinDesc =
|
|
new GlobalVariable(M, getFatbinWrapperTy(M),
|
|
/*isConstant*/ true, GlobalValue::InternalLinkage,
|
|
FatbinInitializer, ".fatbin_wrapper" + Suffix);
|
|
FatbinDesc->setSection(FatbinWrapperSection);
|
|
FatbinDesc->setAlignment(Align(8));
|
|
|
|
return FatbinDesc;
|
|
}
|
|
|
|
/// Create the register globals function. We will iterate all of the offloading
|
|
/// entries stored at the begin / end symbols and register them according to
|
|
/// their type. This creates the following function in IR:
|
|
///
|
|
/// extern struct __tgt_offload_entry __start_cuda_offloading_entries;
|
|
/// extern struct __tgt_offload_entry __stop_cuda_offloading_entries;
|
|
///
|
|
/// extern void __cudaRegisterFunction(void **, void *, void *, void *, int,
|
|
/// void *, void *, void *, void *, int *);
|
|
/// extern void __cudaRegisterVar(void **, void *, void *, void *, int32_t,
|
|
/// int64_t, int32_t, int32_t);
|
|
///
|
|
/// void __cudaRegisterTest(void **fatbinHandle) {
|
|
/// for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries;
|
|
/// entry != &__stop_cuda_offloading_entries; ++entry) {
|
|
/// if (!entry->size)
|
|
/// __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name,
|
|
/// entry->name, -1, 0, 0, 0, 0, 0);
|
|
/// else
|
|
/// __cudaRegisterVar(fatbinHandle, entry->addr, entry->name, entry->name,
|
|
/// 0, entry->size, 0, 0);
|
|
/// }
|
|
/// }
|
|
Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
|
|
EntryArrayTy EntryArray,
|
|
StringRef Suffix,
|
|
bool EmitSurfacesAndTextures) {
|
|
LLVMContext &C = M.getContext();
|
|
auto [EntriesB, EntriesE] = EntryArray;
|
|
|
|
// Get the __cudaRegisterFunction function declaration.
|
|
PointerType *Int8PtrTy = PointerType::get(C, 0);
|
|
PointerType *Int8PtrPtrTy = PointerType::get(C, 0);
|
|
PointerType *Int32PtrTy = PointerType::get(C, 0);
|
|
auto *RegFuncTy = FunctionType::get(
|
|
Type::getInt32Ty(C),
|
|
{Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
|
|
Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy},
|
|
/*isVarArg*/ false);
|
|
FunctionCallee RegFunc = M.getOrInsertFunction(
|
|
IsHIP ? "__hipRegisterFunction" : "__cudaRegisterFunction", RegFuncTy);
|
|
|
|
// Get the __cudaRegisterVar function declaration.
|
|
auto *RegVarTy = FunctionType::get(
|
|
Type::getVoidTy(C),
|
|
{Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
|
|
getSizeTTy(M), Type::getInt32Ty(C), Type::getInt32Ty(C)},
|
|
/*isVarArg*/ false);
|
|
FunctionCallee RegVar = M.getOrInsertFunction(
|
|
IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", RegVarTy);
|
|
|
|
// Get the __cudaRegisterSurface function declaration.
|
|
FunctionType *RegSurfaceTy =
|
|
FunctionType::get(Type::getVoidTy(C),
|
|
{Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
|
|
Type::getInt32Ty(C), Type::getInt32Ty(C)},
|
|
/*isVarArg=*/false);
|
|
FunctionCallee RegSurface = M.getOrInsertFunction(
|
|
IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", RegSurfaceTy);
|
|
|
|
// Get the __cudaRegisterTexture function declaration.
|
|
FunctionType *RegTextureTy = FunctionType::get(
|
|
Type::getVoidTy(C),
|
|
{Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
|
|
Type::getInt32Ty(C), Type::getInt32Ty(C)},
|
|
/*isVarArg=*/false);
|
|
FunctionCallee RegTexture = M.getOrInsertFunction(
|
|
IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", RegTextureTy);
|
|
|
|
auto *RegGlobalsTy = FunctionType::get(Type::getVoidTy(C), Int8PtrPtrTy,
|
|
/*isVarArg*/ false);
|
|
auto *RegGlobalsFn =
|
|
Function::Create(RegGlobalsTy, GlobalValue::InternalLinkage,
|
|
IsHIP ? ".hip.globals_reg" : ".cuda.globals_reg", &M);
|
|
RegGlobalsFn->setSection(".text.startup");
|
|
|
|
// Create the loop to register all the entries.
|
|
IRBuilder<> Builder(BasicBlock::Create(C, "entry", RegGlobalsFn));
|
|
auto *EntryBB = BasicBlock::Create(C, "while.entry", RegGlobalsFn);
|
|
auto *IfThenBB = BasicBlock::Create(C, "if.then", RegGlobalsFn);
|
|
auto *IfElseBB = BasicBlock::Create(C, "if.else", RegGlobalsFn);
|
|
auto *SwGlobalBB = BasicBlock::Create(C, "sw.global", RegGlobalsFn);
|
|
auto *SwManagedBB = BasicBlock::Create(C, "sw.managed", RegGlobalsFn);
|
|
auto *SwSurfaceBB = BasicBlock::Create(C, "sw.surface", RegGlobalsFn);
|
|
auto *SwTextureBB = BasicBlock::Create(C, "sw.texture", RegGlobalsFn);
|
|
auto *IfEndBB = BasicBlock::Create(C, "if.end", RegGlobalsFn);
|
|
auto *ExitBB = BasicBlock::Create(C, "while.end", RegGlobalsFn);
|
|
|
|
auto *EntryCmp = Builder.CreateICmpNE(EntriesB, EntriesE);
|
|
Builder.CreateCondBr(EntryCmp, EntryBB, ExitBB);
|
|
Builder.SetInsertPoint(EntryBB);
|
|
auto *Entry = Builder.CreatePHI(PointerType::getUnqual(C), 2, "entry");
|
|
auto *AddrPtr =
|
|
Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
|
|
{ConstantInt::get(getSizeTTy(M), 0),
|
|
ConstantInt::get(Type::getInt32Ty(C), 0)});
|
|
auto *Addr = Builder.CreateLoad(Int8PtrTy, AddrPtr, "addr");
|
|
auto *NamePtr =
|
|
Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
|
|
{ConstantInt::get(getSizeTTy(M), 0),
|
|
ConstantInt::get(Type::getInt32Ty(C), 1)});
|
|
auto *Name = Builder.CreateLoad(Int8PtrTy, NamePtr, "name");
|
|
auto *SizePtr =
|
|
Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
|
|
{ConstantInt::get(getSizeTTy(M), 0),
|
|
ConstantInt::get(Type::getInt32Ty(C), 2)});
|
|
auto *Size = Builder.CreateLoad(getSizeTTy(M), SizePtr, "size");
|
|
auto *FlagsPtr =
|
|
Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
|
|
{ConstantInt::get(getSizeTTy(M), 0),
|
|
ConstantInt::get(Type::getInt32Ty(C), 3)});
|
|
auto *Flags = Builder.CreateLoad(Type::getInt32Ty(C), FlagsPtr, "flags");
|
|
auto *DataPtr =
|
|
Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
|
|
{ConstantInt::get(getSizeTTy(M), 0),
|
|
ConstantInt::get(Type::getInt32Ty(C), 4)});
|
|
auto *Data = Builder.CreateLoad(Type::getInt32Ty(C), DataPtr, "textype");
|
|
auto *Kind = Builder.CreateAnd(
|
|
Flags, ConstantInt::get(Type::getInt32Ty(C), 0x7), "type");
|
|
|
|
// Extract the flags stored in the bit-field and convert them to C booleans.
|
|
auto *ExternBit = Builder.CreateAnd(
|
|
Flags, ConstantInt::get(Type::getInt32Ty(C),
|
|
llvm::offloading::OffloadGlobalExtern));
|
|
auto *Extern = Builder.CreateLShr(
|
|
ExternBit, ConstantInt::get(Type::getInt32Ty(C), 3), "extern");
|
|
auto *ConstantBit = Builder.CreateAnd(
|
|
Flags, ConstantInt::get(Type::getInt32Ty(C),
|
|
llvm::offloading::OffloadGlobalConstant));
|
|
auto *Const = Builder.CreateLShr(
|
|
ConstantBit, ConstantInt::get(Type::getInt32Ty(C), 4), "constant");
|
|
auto *NormalizedBit = Builder.CreateAnd(
|
|
Flags, ConstantInt::get(Type::getInt32Ty(C),
|
|
llvm::offloading::OffloadGlobalNormalized));
|
|
auto *Normalized = Builder.CreateLShr(
|
|
NormalizedBit, ConstantInt::get(Type::getInt32Ty(C), 5), "normalized");
|
|
auto *FnCond =
|
|
Builder.CreateICmpEQ(Size, ConstantInt::getNullValue(getSizeTTy(M)));
|
|
Builder.CreateCondBr(FnCond, IfThenBB, IfElseBB);
|
|
|
|
// Create kernel registration code.
|
|
Builder.SetInsertPoint(IfThenBB);
|
|
Builder.CreateCall(RegFunc, {RegGlobalsFn->arg_begin(), Addr, Name, Name,
|
|
ConstantInt::get(Type::getInt32Ty(C), -1),
|
|
ConstantPointerNull::get(Int8PtrTy),
|
|
ConstantPointerNull::get(Int8PtrTy),
|
|
ConstantPointerNull::get(Int8PtrTy),
|
|
ConstantPointerNull::get(Int8PtrTy),
|
|
ConstantPointerNull::get(Int32PtrTy)});
|
|
Builder.CreateBr(IfEndBB);
|
|
Builder.SetInsertPoint(IfElseBB);
|
|
|
|
auto *Switch = Builder.CreateSwitch(Kind, IfEndBB);
|
|
// Create global variable registration code.
|
|
Builder.SetInsertPoint(SwGlobalBB);
|
|
Builder.CreateCall(RegVar,
|
|
{RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size,
|
|
Const, ConstantInt::get(Type::getInt32Ty(C), 0)});
|
|
Builder.CreateBr(IfEndBB);
|
|
Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalEntry),
|
|
SwGlobalBB);
|
|
|
|
// Create managed variable registration code.
|
|
Builder.SetInsertPoint(SwManagedBB);
|
|
Builder.CreateBr(IfEndBB);
|
|
Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalManagedEntry),
|
|
SwManagedBB);
|
|
// Create surface variable registration code.
|
|
Builder.SetInsertPoint(SwSurfaceBB);
|
|
if (EmitSurfacesAndTextures)
|
|
Builder.CreateCall(RegSurface, {RegGlobalsFn->arg_begin(), Addr, Name, Name,
|
|
Data, Extern});
|
|
Builder.CreateBr(IfEndBB);
|
|
Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalSurfaceEntry),
|
|
SwSurfaceBB);
|
|
|
|
// Create texture variable registration code.
|
|
Builder.SetInsertPoint(SwTextureBB);
|
|
if (EmitSurfacesAndTextures)
|
|
Builder.CreateCall(RegTexture, {RegGlobalsFn->arg_begin(), Addr, Name, Name,
|
|
Data, Normalized, Extern});
|
|
Builder.CreateBr(IfEndBB);
|
|
Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalTextureEntry),
|
|
SwTextureBB);
|
|
|
|
Builder.SetInsertPoint(IfEndBB);
|
|
auto *NewEntry = Builder.CreateInBoundsGEP(
|
|
offloading::getEntryTy(M), Entry, ConstantInt::get(getSizeTTy(M), 1));
|
|
auto *Cmp = Builder.CreateICmpEQ(
|
|
NewEntry,
|
|
ConstantExpr::getInBoundsGetElementPtr(
|
|
ArrayType::get(offloading::getEntryTy(M), 0), EntriesE,
|
|
ArrayRef<Constant *>({ConstantInt::get(getSizeTTy(M), 0),
|
|
ConstantInt::get(getSizeTTy(M), 0)})));
|
|
Entry->addIncoming(
|
|
ConstantExpr::getInBoundsGetElementPtr(
|
|
ArrayType::get(offloading::getEntryTy(M), 0), EntriesB,
|
|
ArrayRef<Constant *>({ConstantInt::get(getSizeTTy(M), 0),
|
|
ConstantInt::get(getSizeTTy(M), 0)})),
|
|
&RegGlobalsFn->getEntryBlock());
|
|
Entry->addIncoming(NewEntry, IfEndBB);
|
|
Builder.CreateCondBr(Cmp, ExitBB, EntryBB);
|
|
Builder.SetInsertPoint(ExitBB);
|
|
Builder.CreateRetVoid();
|
|
|
|
return RegGlobalsFn;
|
|
}
|
|
|
|
// Create the constructor and destructor to register the fatbinary with the CUDA
|
|
// runtime.
|
|
void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc,
|
|
bool IsHIP, EntryArrayTy EntryArray,
|
|
StringRef Suffix,
|
|
bool EmitSurfacesAndTextures) {
|
|
LLVMContext &C = M.getContext();
|
|
auto *CtorFuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
|
|
auto *CtorFunc = Function::Create(
|
|
CtorFuncTy, GlobalValue::InternalLinkage,
|
|
(IsHIP ? ".hip.fatbin_reg" : ".cuda.fatbin_reg") + Suffix, &M);
|
|
CtorFunc->setSection(".text.startup");
|
|
|
|
auto *DtorFuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
|
|
auto *DtorFunc = Function::Create(
|
|
DtorFuncTy, GlobalValue::InternalLinkage,
|
|
(IsHIP ? ".hip.fatbin_unreg" : ".cuda.fatbin_unreg") + Suffix, &M);
|
|
DtorFunc->setSection(".text.startup");
|
|
|
|
auto *PtrTy = PointerType::getUnqual(C);
|
|
|
|
// Get the __cudaRegisterFatBinary function declaration.
|
|
auto *RegFatTy = FunctionType::get(PtrTy, PtrTy, /*isVarArg=*/false);
|
|
FunctionCallee RegFatbin = M.getOrInsertFunction(
|
|
IsHIP ? "__hipRegisterFatBinary" : "__cudaRegisterFatBinary", RegFatTy);
|
|
// Get the __cudaRegisterFatBinaryEnd function declaration.
|
|
auto *RegFatEndTy =
|
|
FunctionType::get(Type::getVoidTy(C), PtrTy, /*isVarArg=*/false);
|
|
FunctionCallee RegFatbinEnd =
|
|
M.getOrInsertFunction("__cudaRegisterFatBinaryEnd", RegFatEndTy);
|
|
// Get the __cudaUnregisterFatBinary function declaration.
|
|
auto *UnregFatTy =
|
|
FunctionType::get(Type::getVoidTy(C), PtrTy, /*isVarArg=*/false);
|
|
FunctionCallee UnregFatbin = M.getOrInsertFunction(
|
|
IsHIP ? "__hipUnregisterFatBinary" : "__cudaUnregisterFatBinary",
|
|
UnregFatTy);
|
|
|
|
auto *AtExitTy =
|
|
FunctionType::get(Type::getInt32Ty(C), PtrTy, /*isVarArg=*/false);
|
|
FunctionCallee AtExit = M.getOrInsertFunction("atexit", AtExitTy);
|
|
|
|
auto *BinaryHandleGlobal = new llvm::GlobalVariable(
|
|
M, PtrTy, false, llvm::GlobalValue::InternalLinkage,
|
|
llvm::ConstantPointerNull::get(PtrTy),
|
|
(IsHIP ? ".hip.binary_handle" : ".cuda.binary_handle") + Suffix);
|
|
|
|
// Create the constructor to register this image with the runtime.
|
|
IRBuilder<> CtorBuilder(BasicBlock::Create(C, "entry", CtorFunc));
|
|
CallInst *Handle = CtorBuilder.CreateCall(
|
|
RegFatbin,
|
|
ConstantExpr::getPointerBitCastOrAddrSpaceCast(FatbinDesc, PtrTy));
|
|
CtorBuilder.CreateAlignedStore(
|
|
Handle, BinaryHandleGlobal,
|
|
Align(M.getDataLayout().getPointerTypeSize(PtrTy)));
|
|
CtorBuilder.CreateCall(createRegisterGlobalsFunction(M, IsHIP, EntryArray,
|
|
Suffix,
|
|
EmitSurfacesAndTextures),
|
|
Handle);
|
|
if (!IsHIP)
|
|
CtorBuilder.CreateCall(RegFatbinEnd, Handle);
|
|
CtorBuilder.CreateCall(AtExit, DtorFunc);
|
|
CtorBuilder.CreateRetVoid();
|
|
|
|
// Create the destructor to unregister the image with the runtime. We cannot
|
|
// use a standard global destructor after CUDA 9.2 so this must be called by
|
|
// `atexit()` intead.
|
|
IRBuilder<> DtorBuilder(BasicBlock::Create(C, "entry", DtorFunc));
|
|
LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad(
|
|
PtrTy, BinaryHandleGlobal,
|
|
Align(M.getDataLayout().getPointerTypeSize(PtrTy)));
|
|
DtorBuilder.CreateCall(UnregFatbin, BinaryHandle);
|
|
DtorBuilder.CreateRetVoid();
|
|
|
|
// Add this function to constructors.
|
|
appendToGlobalCtors(M, CtorFunc, /*Priority*/ 1);
|
|
}
|
|
} // namespace
|
|
|
|
Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images,
|
|
EntryArrayTy EntryArray,
|
|
llvm::StringRef Suffix, bool Relocatable) {
|
|
GlobalVariable *Desc =
|
|
createBinDesc(M, Images, EntryArray, Suffix, Relocatable);
|
|
if (!Desc)
|
|
return createStringError(inconvertibleErrorCode(),
|
|
"No binary descriptors created.");
|
|
createRegisterFunction(M, Desc, Suffix);
|
|
createUnregisterFunction(M, Desc, Suffix);
|
|
return Error::success();
|
|
}
|
|
|
|
Error offloading::wrapCudaBinary(Module &M, ArrayRef<char> Image,
|
|
EntryArrayTy EntryArray,
|
|
llvm::StringRef Suffix,
|
|
bool EmitSurfacesAndTextures) {
|
|
GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/false, Suffix);
|
|
if (!Desc)
|
|
return createStringError(inconvertibleErrorCode(),
|
|
"No fatbin section created.");
|
|
|
|
createRegisterFatbinFunction(M, Desc, /*IsHip=*/false, EntryArray, Suffix,
|
|
EmitSurfacesAndTextures);
|
|
return Error::success();
|
|
}
|
|
|
|
Error offloading::wrapHIPBinary(Module &M, ArrayRef<char> Image,
|
|
EntryArrayTy EntryArray, llvm::StringRef Suffix,
|
|
bool EmitSurfacesAndTextures) {
|
|
GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/true, Suffix);
|
|
if (!Desc)
|
|
return createStringError(inconvertibleErrorCode(),
|
|
"No fatbin section created.");
|
|
|
|
createRegisterFatbinFunction(M, Desc, /*IsHip=*/true, EntryArray, Suffix,
|
|
EmitSurfacesAndTextures);
|
|
return Error::success();
|
|
}
|