15218a1a48
This patch adds resolver, indirection and trampoline stubs for loongarch64, allowing lazy compilation to work. It assumes hard float feature exists. Depends on D141036 Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D141102
433 lines
13 KiB
C++
433 lines
13 KiB
C++
//===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
|
|
|
|
#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
|
|
#include "llvm/Support/MathExtras.h"
|
|
|
|
#include <future>
|
|
|
|
using namespace llvm;
|
|
using namespace llvm::orc;
|
|
|
|
namespace llvm {
|
|
namespace orc {
|
|
|
|
class EPCIndirectionUtilsAccess {
|
|
public:
|
|
using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
|
|
using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
|
|
|
|
static Expected<IndirectStubInfoVector>
|
|
getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
|
|
return EPCIU.getIndirectStubs(NumStubs);
|
|
};
|
|
};
|
|
|
|
} // end namespace orc
|
|
} // end namespace llvm
|
|
|
|
namespace {
|
|
|
|
class EPCTrampolinePool : public TrampolinePool {
|
|
public:
|
|
EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
|
|
Error deallocatePool();
|
|
|
|
protected:
|
|
Error grow() override;
|
|
|
|
using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
|
|
|
|
EPCIndirectionUtils &EPCIU;
|
|
unsigned TrampolineSize = 0;
|
|
unsigned TrampolinesPerPage = 0;
|
|
std::vector<FinalizedAlloc> TrampolineBlocks;
|
|
};
|
|
|
|
class EPCIndirectStubsManager : public IndirectStubsManager,
|
|
private EPCIndirectionUtilsAccess {
|
|
public:
|
|
EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
|
|
|
|
Error deallocateStubs();
|
|
|
|
Error createStub(StringRef StubName, JITTargetAddress StubAddr,
|
|
JITSymbolFlags StubFlags) override;
|
|
|
|
Error createStubs(const StubInitsMap &StubInits) override;
|
|
|
|
JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
|
|
|
|
JITEvaluatedSymbol findPointer(StringRef Name) override;
|
|
|
|
Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override;
|
|
|
|
private:
|
|
using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
|
|
|
|
std::mutex ISMMutex;
|
|
EPCIndirectionUtils &EPCIU;
|
|
StringMap<StubInfo> StubInfos;
|
|
};
|
|
|
|
EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
|
|
: EPCIU(EPCIU) {
|
|
auto &EPC = EPCIU.getExecutorProcessControl();
|
|
auto &ABI = EPCIU.getABISupport();
|
|
|
|
TrampolineSize = ABI.getTrampolineSize();
|
|
TrampolinesPerPage =
|
|
(EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
|
|
}
|
|
|
|
Error EPCTrampolinePool::deallocatePool() {
|
|
std::promise<MSVCPError> DeallocResultP;
|
|
auto DeallocResultF = DeallocResultP.get_future();
|
|
|
|
EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
|
|
std::move(TrampolineBlocks),
|
|
[&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
|
|
|
|
return DeallocResultF.get();
|
|
}
|
|
|
|
Error EPCTrampolinePool::grow() {
|
|
using namespace jitlink;
|
|
|
|
assert(AvailableTrampolines.empty() &&
|
|
"Grow called with trampolines still available");
|
|
|
|
auto ResolverAddress = EPCIU.getResolverBlockAddress();
|
|
assert(ResolverAddress && "Resolver address can not be null");
|
|
|
|
auto &EPC = EPCIU.getExecutorProcessControl();
|
|
auto PageSize = EPC.getPageSize();
|
|
auto Alloc = SimpleSegmentAlloc::Create(
|
|
EPC.getMemMgr(), nullptr,
|
|
{{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
|
|
if (!Alloc)
|
|
return Alloc.takeError();
|
|
|
|
unsigned NumTrampolines = TrampolinesPerPage;
|
|
|
|
auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
|
|
EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(),
|
|
SegInfo.Addr.getValue(),
|
|
ResolverAddress, NumTrampolines);
|
|
for (unsigned I = 0; I < NumTrampolines; ++I)
|
|
AvailableTrampolines.push_back(SegInfo.Addr.getValue() +
|
|
(I * TrampolineSize));
|
|
|
|
auto FA = Alloc->finalize();
|
|
if (!FA)
|
|
return FA.takeError();
|
|
|
|
TrampolineBlocks.push_back(std::move(*FA));
|
|
|
|
return Error::success();
|
|
}
|
|
|
|
Error EPCIndirectStubsManager::createStub(StringRef StubName,
|
|
JITTargetAddress StubAddr,
|
|
JITSymbolFlags StubFlags) {
|
|
StubInitsMap SIM;
|
|
SIM[StubName] = std::make_pair(StubAddr, StubFlags);
|
|
return createStubs(SIM);
|
|
}
|
|
|
|
Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
|
|
auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
|
|
if (!AvailableStubInfos)
|
|
return AvailableStubInfos.takeError();
|
|
|
|
{
|
|
std::lock_guard<std::mutex> Lock(ISMMutex);
|
|
unsigned ASIdx = 0;
|
|
for (auto &SI : StubInits) {
|
|
auto &A = (*AvailableStubInfos)[ASIdx++];
|
|
StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
|
|
}
|
|
}
|
|
|
|
auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
|
|
switch (EPCIU.getABISupport().getPointerSize()) {
|
|
case 4: {
|
|
unsigned ASIdx = 0;
|
|
std::vector<tpctypes::UInt32Write> PtrUpdates;
|
|
for (auto &SI : StubInits)
|
|
PtrUpdates.push_back(
|
|
{ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
|
|
static_cast<uint32_t>(SI.second.first)});
|
|
return MemAccess.writeUInt32s(PtrUpdates);
|
|
}
|
|
case 8: {
|
|
unsigned ASIdx = 0;
|
|
std::vector<tpctypes::UInt64Write> PtrUpdates;
|
|
for (auto &SI : StubInits)
|
|
PtrUpdates.push_back(
|
|
{ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
|
|
static_cast<uint64_t>(SI.second.first)});
|
|
return MemAccess.writeUInt64s(PtrUpdates);
|
|
}
|
|
default:
|
|
return make_error<StringError>("Unsupported pointer size",
|
|
inconvertibleErrorCode());
|
|
}
|
|
}
|
|
|
|
JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name,
|
|
bool ExportedStubsOnly) {
|
|
std::lock_guard<std::mutex> Lock(ISMMutex);
|
|
auto I = StubInfos.find(Name);
|
|
if (I == StubInfos.end())
|
|
return nullptr;
|
|
return {I->second.first.StubAddress, I->second.second};
|
|
}
|
|
|
|
JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) {
|
|
std::lock_guard<std::mutex> Lock(ISMMutex);
|
|
auto I = StubInfos.find(Name);
|
|
if (I == StubInfos.end())
|
|
return nullptr;
|
|
return {I->second.first.PointerAddress, I->second.second};
|
|
}
|
|
|
|
Error EPCIndirectStubsManager::updatePointer(StringRef Name,
|
|
JITTargetAddress NewAddr) {
|
|
|
|
JITTargetAddress PtrAddr = 0;
|
|
{
|
|
std::lock_guard<std::mutex> Lock(ISMMutex);
|
|
auto I = StubInfos.find(Name);
|
|
if (I == StubInfos.end())
|
|
return make_error<StringError>("Unknown stub name",
|
|
inconvertibleErrorCode());
|
|
PtrAddr = I->second.first.PointerAddress;
|
|
}
|
|
|
|
auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
|
|
switch (EPCIU.getABISupport().getPointerSize()) {
|
|
case 4: {
|
|
tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
|
|
return MemAccess.writeUInt32s(PUpdate);
|
|
}
|
|
case 8: {
|
|
tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
|
|
return MemAccess.writeUInt64s(PUpdate);
|
|
}
|
|
default:
|
|
return make_error<StringError>("Unsupported pointer size",
|
|
inconvertibleErrorCode());
|
|
}
|
|
}
|
|
|
|
} // end anonymous namespace.
|
|
|
|
namespace llvm {
|
|
namespace orc {
|
|
|
|
EPCIndirectionUtils::ABISupport::~ABISupport() = default;
|
|
|
|
Expected<std::unique_ptr<EPCIndirectionUtils>>
|
|
EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
|
|
const auto &TT = EPC.getTargetTriple();
|
|
switch (TT.getArch()) {
|
|
default:
|
|
return make_error<StringError>(
|
|
std::string("No EPCIndirectionUtils available for ") + TT.str(),
|
|
inconvertibleErrorCode());
|
|
case Triple::aarch64:
|
|
case Triple::aarch64_32:
|
|
return CreateWithABI<OrcAArch64>(EPC);
|
|
|
|
case Triple::x86:
|
|
return CreateWithABI<OrcI386>(EPC);
|
|
|
|
case Triple::loongarch64:
|
|
return CreateWithABI<OrcLoongArch64>(EPC);
|
|
|
|
case Triple::mips:
|
|
return CreateWithABI<OrcMips32Be>(EPC);
|
|
|
|
case Triple::mipsel:
|
|
return CreateWithABI<OrcMips32Le>(EPC);
|
|
|
|
case Triple::mips64:
|
|
case Triple::mips64el:
|
|
return CreateWithABI<OrcMips64>(EPC);
|
|
|
|
case Triple::riscv64:
|
|
return CreateWithABI<OrcRiscv64>(EPC);
|
|
|
|
case Triple::x86_64:
|
|
if (TT.getOS() == Triple::OSType::Win32)
|
|
return CreateWithABI<OrcX86_64_Win32>(EPC);
|
|
else
|
|
return CreateWithABI<OrcX86_64_SysV>(EPC);
|
|
}
|
|
}
|
|
|
|
Error EPCIndirectionUtils::cleanup() {
|
|
|
|
auto &MemMgr = EPC.getMemMgr();
|
|
auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
|
|
|
|
if (TP)
|
|
Err = joinErrors(std::move(Err),
|
|
static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
|
|
|
|
if (ResolverBlock)
|
|
Err =
|
|
joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
|
|
|
|
return Err;
|
|
}
|
|
|
|
Expected<JITTargetAddress>
|
|
EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
|
|
JITTargetAddress ReentryCtxAddr) {
|
|
using namespace jitlink;
|
|
|
|
assert(ABI && "ABI can not be null");
|
|
auto ResolverSize = ABI->getResolverCodeSize();
|
|
|
|
auto Alloc =
|
|
SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
|
|
{{MemProt::Read | MemProt::Exec,
|
|
{ResolverSize, Align(EPC.getPageSize())}}});
|
|
|
|
if (!Alloc)
|
|
return Alloc.takeError();
|
|
|
|
auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
|
|
ResolverBlockAddr = SegInfo.Addr.getValue();
|
|
ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,
|
|
ReentryFnAddr, ReentryCtxAddr);
|
|
|
|
auto FA = Alloc->finalize();
|
|
if (!FA)
|
|
return FA.takeError();
|
|
|
|
ResolverBlock = std::move(*FA);
|
|
return ResolverBlockAddr;
|
|
}
|
|
|
|
std::unique_ptr<IndirectStubsManager>
|
|
EPCIndirectionUtils::createIndirectStubsManager() {
|
|
return std::make_unique<EPCIndirectStubsManager>(*this);
|
|
}
|
|
|
|
TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
|
|
if (!TP)
|
|
TP = std::make_unique<EPCTrampolinePool>(*this);
|
|
return *TP;
|
|
}
|
|
|
|
LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
|
|
ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
|
|
assert(!LCTM &&
|
|
"createLazyCallThroughManager can not have been called before");
|
|
LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
|
|
&getTrampolinePool());
|
|
return *LCTM;
|
|
}
|
|
|
|
EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
|
|
std::unique_ptr<ABISupport> ABI)
|
|
: EPC(EPC), ABI(std::move(ABI)) {
|
|
assert(this->ABI && "ABI can not be null");
|
|
|
|
assert(EPC.getPageSize() > getABISupport().getStubSize() &&
|
|
"Stubs larger than one page are not supported");
|
|
}
|
|
|
|
Expected<EPCIndirectionUtils::IndirectStubInfoVector>
|
|
EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
|
|
using namespace jitlink;
|
|
|
|
std::lock_guard<std::mutex> Lock(EPCUIMutex);
|
|
|
|
// If there aren't enough stubs available then allocate some more.
|
|
if (NumStubs > AvailableIndirectStubs.size()) {
|
|
auto NumStubsToAllocate = NumStubs;
|
|
auto PageSize = EPC.getPageSize();
|
|
auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
|
|
NumStubsToAllocate = StubBytes / ABI->getStubSize();
|
|
auto PtrBytes =
|
|
alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
|
|
|
|
auto StubProt = MemProt::Read | MemProt::Exec;
|
|
auto PtrProt = MemProt::Read | MemProt::Write;
|
|
|
|
auto Alloc = SimpleSegmentAlloc::Create(
|
|
EPC.getMemMgr(), nullptr,
|
|
{{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
|
|
{PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
|
|
|
|
if (!Alloc)
|
|
return Alloc.takeError();
|
|
|
|
auto StubSeg = Alloc->getSegInfo(StubProt);
|
|
auto PtrSeg = Alloc->getSegInfo(PtrProt);
|
|
|
|
ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(),
|
|
StubSeg.Addr.getValue(),
|
|
PtrSeg.Addr.getValue(), NumStubsToAllocate);
|
|
|
|
auto FA = Alloc->finalize();
|
|
if (!FA)
|
|
return FA.takeError();
|
|
|
|
IndirectStubAllocs.push_back(std::move(*FA));
|
|
|
|
auto StubExecutorAddr = StubSeg.Addr;
|
|
auto PtrExecutorAddr = PtrSeg.Addr;
|
|
for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
|
|
AvailableIndirectStubs.push_back(IndirectStubInfo(
|
|
StubExecutorAddr.getValue(), PtrExecutorAddr.getValue()));
|
|
StubExecutorAddr += ABI->getStubSize();
|
|
PtrExecutorAddr += ABI->getPointerSize();
|
|
}
|
|
}
|
|
|
|
assert(NumStubs <= AvailableIndirectStubs.size() &&
|
|
"Sufficient stubs should have been allocated above");
|
|
|
|
IndirectStubInfoVector Result;
|
|
while (NumStubs--) {
|
|
Result.push_back(AvailableIndirectStubs.back());
|
|
AvailableIndirectStubs.pop_back();
|
|
}
|
|
|
|
return std::move(Result);
|
|
}
|
|
|
|
static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
|
|
JITTargetAddress TrampolineAddr) {
|
|
auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
|
|
std::promise<JITTargetAddress> LandingAddrP;
|
|
auto LandingAddrF = LandingAddrP.get_future();
|
|
LCTM.resolveTrampolineLandingAddress(
|
|
TrampolineAddr,
|
|
[&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
|
|
return LandingAddrF.get();
|
|
}
|
|
|
|
Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
|
|
auto &LCTM = EPCIU.getLazyCallThroughManager();
|
|
return EPCIU
|
|
.writeResolverBlock(pointerToJITTargetAddress(&reentry),
|
|
pointerToJITTargetAddress(&LCTM))
|
|
.takeError();
|
|
}
|
|
|
|
} // end namespace orc
|
|
} // end namespace llvm
|