4374ca3cab
Teach ExecutorSimpleMemoryManager to handle slab reserve/release operations, plus separate initialize/deinitialize for regions within the slab. The release operation automatically deinitializes any regions within each slab that have not already been released. EPCGenericJITLinkMemoryManager is updated to use the reserve (allocate), initialize (finalize), and relesae (deallocate) operations. This brings ExecutorSimpleMemoryManager into alignment with the orc_rt::SimpleNativeMemoryMap class, allowing SimpleNativeMemoryMap to be used as a backend for EPCGenericJITLinkMemoryManager. A future commit will introduce a new MemoryMapper class that will make SimpleNativeMemoryMap usable as a backend for MapperJITLinkMemoryManager. This work will make it easier to re-use in-tree APIs and tools with the new ORC runtime.
316 lines
12 KiB
C++
316 lines
12 KiB
C++
//===----- EPCGenericRTDyldMemoryManager.cpp - EPC-bbasde MemMgr -----===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h"
|
|
#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
|
|
#include "llvm/Support/Alignment.h"
|
|
#include "llvm/Support/FormatVariadic.h"
|
|
|
|
#define DEBUG_TYPE "orc"
|
|
|
|
using namespace llvm::orc::shared;
|
|
|
|
namespace llvm {
|
|
namespace orc {
|
|
|
|
Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>>
|
|
EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols(
|
|
ExecutorProcessControl &EPC) {
|
|
SymbolAddrs SAs;
|
|
if (auto Err = EPC.getBootstrapSymbols(
|
|
{{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName},
|
|
{SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName},
|
|
{SAs.Initialize,
|
|
rt::SimpleExecutorMemoryManagerInitializeWrapperName},
|
|
{SAs.Release, rt::SimpleExecutorMemoryManagerReleaseWrapperName},
|
|
{SAs.RegisterEHFrame, rt::RegisterEHFrameSectionAllocActionName},
|
|
{SAs.DeregisterEHFrame,
|
|
rt::DeregisterEHFrameSectionAllocActionName}}))
|
|
return std::move(Err);
|
|
return std::make_unique<EPCGenericRTDyldMemoryManager>(EPC, std::move(SAs));
|
|
}
|
|
|
|
EPCGenericRTDyldMemoryManager::EPCGenericRTDyldMemoryManager(
|
|
ExecutorProcessControl &EPC, SymbolAddrs SAs)
|
|
: EPC(EPC), SAs(std::move(SAs)) {
|
|
LLVM_DEBUG(dbgs() << "Created remote allocator " << (void *)this << "\n");
|
|
}
|
|
|
|
EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() {
|
|
LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << (void *)this << "\n");
|
|
if (!ErrMsg.empty())
|
|
errs() << "Destroying with existing errors:\n" << ErrMsg << "\n";
|
|
|
|
Error Err = Error::success();
|
|
if (auto Err2 = EPC.callSPSWrapper<
|
|
rt::SPSSimpleExecutorMemoryManagerReleaseSignature>(
|
|
SAs.Reserve, Err, SAs.Instance, FinalizedAllocs)) {
|
|
// FIXME: Report errors through EPC once that functionality is available.
|
|
logAllUnhandledErrors(std::move(Err2), errs(), "");
|
|
return;
|
|
}
|
|
|
|
if (Err)
|
|
logAllUnhandledErrors(std::move(Err), errs(), "");
|
|
}
|
|
|
|
uint8_t *EPCGenericRTDyldMemoryManager::allocateCodeSection(
|
|
uintptr_t Size, unsigned Alignment, unsigned SectionID,
|
|
StringRef SectionName) {
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
LLVM_DEBUG({
|
|
dbgs() << "Allocator " << (void *)this << " allocating code section "
|
|
<< SectionName << ": size = " << formatv("{0:x}", Size)
|
|
<< " bytes, alignment = " << Alignment << "\n";
|
|
});
|
|
auto &Seg = Unmapped.back().CodeAllocs;
|
|
Seg.emplace_back(Size, Alignment);
|
|
return reinterpret_cast<uint8_t *>(
|
|
alignAddr(Seg.back().Contents.get(), Align(Alignment)));
|
|
}
|
|
|
|
uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection(
|
|
uintptr_t Size, unsigned Alignment, unsigned SectionID,
|
|
StringRef SectionName, bool IsReadOnly) {
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
LLVM_DEBUG({
|
|
dbgs() << "Allocator " << (void *)this << " allocating "
|
|
<< (IsReadOnly ? "ro" : "rw") << "-data section " << SectionName
|
|
<< ": size = " << formatv("{0:x}", Size) << " bytes, alignment "
|
|
<< Alignment << ")\n";
|
|
});
|
|
|
|
auto &Seg =
|
|
IsReadOnly ? Unmapped.back().RODataAllocs : Unmapped.back().RWDataAllocs;
|
|
|
|
Seg.emplace_back(Size, Alignment);
|
|
return reinterpret_cast<uint8_t *>(
|
|
alignAddr(Seg.back().Contents.get(), Align(Alignment)));
|
|
}
|
|
|
|
void EPCGenericRTDyldMemoryManager::reserveAllocationSpace(
|
|
uintptr_t CodeSize, Align CodeAlign, uintptr_t RODataSize,
|
|
Align RODataAlign, uintptr_t RWDataSize, Align RWDataAlign) {
|
|
|
|
{
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
// If there's already an error then bail out.
|
|
if (!ErrMsg.empty())
|
|
return;
|
|
|
|
if (CodeAlign > EPC.getPageSize()) {
|
|
ErrMsg = "Invalid code alignment in reserveAllocationSpace";
|
|
return;
|
|
}
|
|
if (RODataAlign > EPC.getPageSize()) {
|
|
ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace";
|
|
return;
|
|
}
|
|
if (RWDataAlign > EPC.getPageSize()) {
|
|
ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace";
|
|
return;
|
|
}
|
|
}
|
|
|
|
uint64_t TotalSize = 0;
|
|
TotalSize += alignTo(CodeSize, EPC.getPageSize());
|
|
TotalSize += alignTo(RODataSize, EPC.getPageSize());
|
|
TotalSize += alignTo(RWDataSize, EPC.getPageSize());
|
|
|
|
LLVM_DEBUG({
|
|
dbgs() << "Allocator " << (void *)this << " reserving "
|
|
<< formatv("{0:x}", TotalSize) << " bytes.\n";
|
|
});
|
|
|
|
Expected<ExecutorAddr> TargetAllocAddr((ExecutorAddr()));
|
|
if (auto Err = EPC.callSPSWrapper<
|
|
rt::SPSSimpleExecutorMemoryManagerReserveSignature>(
|
|
SAs.Reserve, TargetAllocAddr, SAs.Instance, TotalSize)) {
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
ErrMsg = toString(std::move(Err));
|
|
return;
|
|
}
|
|
if (!TargetAllocAddr) {
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
ErrMsg = toString(TargetAllocAddr.takeError());
|
|
return;
|
|
}
|
|
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
Unmapped.push_back(SectionAllocGroup());
|
|
Unmapped.back().RemoteCode = {
|
|
*TargetAllocAddr, ExecutorAddrDiff(alignTo(CodeSize, EPC.getPageSize()))};
|
|
Unmapped.back().RemoteROData = {
|
|
Unmapped.back().RemoteCode.End,
|
|
ExecutorAddrDiff(alignTo(RODataSize, EPC.getPageSize()))};
|
|
Unmapped.back().RemoteRWData = {
|
|
Unmapped.back().RemoteROData.End,
|
|
ExecutorAddrDiff(alignTo(RWDataSize, EPC.getPageSize()))};
|
|
}
|
|
|
|
bool EPCGenericRTDyldMemoryManager::needsToReserveAllocationSpace() {
|
|
return true;
|
|
}
|
|
|
|
void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr,
|
|
uint64_t LoadAddr,
|
|
size_t Size) {
|
|
LLVM_DEBUG({
|
|
dbgs() << "Allocator " << (void *)this << " added unfinalized eh-frame "
|
|
<< formatv("[ {0:x} {1:x} ]", LoadAddr, LoadAddr + Size) << "\n";
|
|
});
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
// Bail out early if there's already an error.
|
|
if (!ErrMsg.empty())
|
|
return;
|
|
|
|
ExecutorAddr LA(LoadAddr);
|
|
for (auto &SecAllocGroup : llvm::reverse(Unfinalized)) {
|
|
if (SecAllocGroup.RemoteCode.contains(LA) ||
|
|
SecAllocGroup.RemoteROData.contains(LA) ||
|
|
SecAllocGroup.RemoteRWData.contains(LA)) {
|
|
SecAllocGroup.UnfinalizedEHFrames.push_back({LA, Size});
|
|
return;
|
|
}
|
|
}
|
|
ErrMsg = "eh-frame does not lie inside unfinalized alloc";
|
|
}
|
|
|
|
void EPCGenericRTDyldMemoryManager::deregisterEHFrames() {
|
|
// This is a no-op for us: We've registered a deallocation action for it.
|
|
}
|
|
|
|
void EPCGenericRTDyldMemoryManager::notifyObjectLoaded(
|
|
RuntimeDyld &Dyld, const object::ObjectFile &Obj) {
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " applied mappings:\n");
|
|
for (auto &ObjAllocs : Unmapped) {
|
|
mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs,
|
|
ObjAllocs.RemoteCode.Start);
|
|
mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RODataAllocs,
|
|
ObjAllocs.RemoteROData.Start);
|
|
mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RWDataAllocs,
|
|
ObjAllocs.RemoteRWData.Start);
|
|
Unfinalized.push_back(std::move(ObjAllocs));
|
|
}
|
|
Unmapped.clear();
|
|
}
|
|
|
|
bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) {
|
|
LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n");
|
|
|
|
// If there's an error then bail out here.
|
|
std::vector<SectionAllocGroup> SecAllocGroups;
|
|
{
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
if (ErrMsg && !this->ErrMsg.empty()) {
|
|
*ErrMsg = std::move(this->ErrMsg);
|
|
return true;
|
|
}
|
|
std::swap(SecAllocGroups, Unfinalized);
|
|
}
|
|
|
|
// Loop over unfinalized objects to make finalization requests.
|
|
for (auto &SecAllocGroup : SecAllocGroups) {
|
|
|
|
MemProt SegMemProts[3] = {MemProt::Read | MemProt::Exec, MemProt::Read,
|
|
MemProt::Read | MemProt::Write};
|
|
|
|
ExecutorAddrRange *RemoteAddrs[3] = {&SecAllocGroup.RemoteCode,
|
|
&SecAllocGroup.RemoteROData,
|
|
&SecAllocGroup.RemoteRWData};
|
|
|
|
std::vector<SectionAlloc> *SegSections[3] = {&SecAllocGroup.CodeAllocs,
|
|
&SecAllocGroup.RODataAllocs,
|
|
&SecAllocGroup.RWDataAllocs};
|
|
|
|
tpctypes::FinalizeRequest FR;
|
|
std::unique_ptr<char[]> AggregateContents[3];
|
|
|
|
for (unsigned I = 0; I != 3; ++I) {
|
|
FR.Segments.push_back({});
|
|
auto &Seg = FR.Segments.back();
|
|
Seg.RAG = SegMemProts[I];
|
|
Seg.Addr = RemoteAddrs[I]->Start;
|
|
for (auto &SecAlloc : *SegSections[I]) {
|
|
Seg.Size = alignTo(Seg.Size, SecAlloc.Align);
|
|
Seg.Size += SecAlloc.Size;
|
|
}
|
|
AggregateContents[I] = std::make_unique<char[]>(Seg.Size);
|
|
size_t SecOffset = 0;
|
|
for (auto &SecAlloc : *SegSections[I]) {
|
|
SecOffset = alignTo(SecOffset, SecAlloc.Align);
|
|
memcpy(&AggregateContents[I][SecOffset],
|
|
reinterpret_cast<const char *>(
|
|
alignAddr(SecAlloc.Contents.get(), Align(SecAlloc.Align))),
|
|
SecAlloc.Size);
|
|
SecOffset += SecAlloc.Size;
|
|
// FIXME: Can we reset SecAlloc.Content here, now that it's copied into
|
|
// the aggregated content?
|
|
}
|
|
Seg.Content = {AggregateContents[I].get(), SecOffset};
|
|
}
|
|
|
|
for (auto &Frame : SecAllocGroup.UnfinalizedEHFrames)
|
|
FR.Actions.push_back(
|
|
{cantFail(
|
|
WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
|
|
SAs.RegisterEHFrame, Frame)),
|
|
cantFail(
|
|
WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
|
|
SAs.DeregisterEHFrame, Frame))});
|
|
|
|
// We'll also need to make an extra allocation for the eh-frame wrapper call
|
|
// arguments.
|
|
Expected<ExecutorAddr> InitializeKey((ExecutorAddr()));
|
|
if (auto Err = EPC.callSPSWrapper<
|
|
rt::SPSSimpleExecutorMemoryManagerInitializeSignature>(
|
|
SAs.Initialize, InitializeKey, SAs.Instance, std::move(FR))) {
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
this->ErrMsg = toString(std::move(Err));
|
|
dbgs() << "Serialization error: " << this->ErrMsg << "\n";
|
|
if (ErrMsg)
|
|
*ErrMsg = this->ErrMsg;
|
|
return true;
|
|
}
|
|
if (!InitializeKey) {
|
|
std::lock_guard<std::mutex> Lock(M);
|
|
this->ErrMsg = toString(InitializeKey.takeError());
|
|
dbgs() << "Finalization error: " << this->ErrMsg << "\n";
|
|
if (ErrMsg)
|
|
*ErrMsg = this->ErrMsg;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs(
|
|
RuntimeDyld &Dyld, std::vector<SectionAlloc> &Allocs,
|
|
ExecutorAddr NextAddr) {
|
|
for (auto &Alloc : Allocs) {
|
|
NextAddr.setValue(alignTo(NextAddr.getValue(), Alloc.Align));
|
|
LLVM_DEBUG({
|
|
dbgs() << " " << static_cast<void *>(Alloc.Contents.get()) << " -> "
|
|
<< format("0x%016" PRIx64, NextAddr.getValue()) << "\n";
|
|
});
|
|
Dyld.mapSectionAddress(reinterpret_cast<const void *>(alignAddr(
|
|
Alloc.Contents.get(), Align(Alloc.Align))),
|
|
NextAddr.getValue());
|
|
Alloc.RemoteAddr = NextAddr;
|
|
// Only advance NextAddr if it was non-null to begin with,
|
|
// otherwise leave it as null.
|
|
if (NextAddr)
|
|
NextAddr += ExecutorAddrDiff(Alloc.Size);
|
|
}
|
|
}
|
|
|
|
} // end namespace orc
|
|
} // end namespace llvm
|