2946cd7010
to reflect the new license. We understand that people may be surprised that we're moving the header entirely to discuss the new license. We checked this carefully with the Foundation's lawyer and we believe this is the correct approach. Essentially, all code in the project is now made available by the LLVM project under our new license, so you will see that the license headers include that license only. Some of our contributors have contributed code under our old license, and accordingly, we have retained a copy of our old license notice in the top-level files in each project and repository. llvm-svn: 351636
533 lines
18 KiB
C++
533 lines
18 KiB
C++
//===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
|
|
#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
|
|
|
|
#include "llvm-c/OrcBindings.h"
|
|
#include "llvm-c/TargetMachine.h"
|
|
#include "llvm/ADT/STLExtras.h"
|
|
#include "llvm/ADT/StringRef.h"
|
|
#include "llvm/ExecutionEngine/JITSymbol.h"
|
|
#include "llvm/ExecutionEngine/JITEventListener.h"
|
|
#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
|
|
#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
|
|
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
|
|
#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
|
|
#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
|
|
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
|
|
#include "llvm/ExecutionEngine/RuntimeDyld.h"
|
|
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
|
|
#include "llvm/IR/DataLayout.h"
|
|
#include "llvm/IR/Mangler.h"
|
|
#include "llvm/IR/Module.h"
|
|
#include "llvm/Support/CBindingWrapping.h"
|
|
#include "llvm/Support/Error.h"
|
|
#include "llvm/Support/raw_ostream.h"
|
|
#include "llvm/Target/TargetMachine.h"
|
|
#include <algorithm>
|
|
#include <cstdint>
|
|
#include <functional>
|
|
#include <map>
|
|
#include <memory>
|
|
#include <set>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
namespace llvm {
|
|
|
|
class OrcCBindingsStack;
|
|
|
|
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
|
|
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
|
|
|
|
namespace detail {
|
|
|
|
// FIXME: Kill this off once the Layer concept becomes an interface.
|
|
class GenericLayer {
|
|
public:
|
|
virtual ~GenericLayer() = default;
|
|
|
|
virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
|
|
bool ExportedSymbolsOnly) = 0;
|
|
virtual Error removeModule(orc::VModuleKey K) = 0;
|
|
};
|
|
|
|
template <typename LayerT> class GenericLayerImpl : public GenericLayer {
|
|
public:
|
|
GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
|
|
|
|
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
|
|
bool ExportedSymbolsOnly) override {
|
|
return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
|
|
}
|
|
|
|
Error removeModule(orc::VModuleKey K) override {
|
|
return Layer.removeModule(K);
|
|
}
|
|
|
|
private:
|
|
LayerT &Layer;
|
|
};
|
|
|
|
template <>
|
|
class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer {
|
|
private:
|
|
using LayerT = orc::LegacyRTDyldObjectLinkingLayer;
|
|
public:
|
|
GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
|
|
|
|
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
|
|
bool ExportedSymbolsOnly) override {
|
|
return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
|
|
}
|
|
|
|
Error removeModule(orc::VModuleKey K) override {
|
|
return Layer.removeObject(K);
|
|
}
|
|
|
|
private:
|
|
LayerT &Layer;
|
|
};
|
|
|
|
template <typename LayerT>
|
|
std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
|
|
return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer);
|
|
}
|
|
|
|
} // end namespace detail
|
|
|
|
class OrcCBindingsStack {
|
|
public:
|
|
|
|
using CompileCallbackMgr = orc::JITCompileCallbackManager;
|
|
using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer;
|
|
using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
|
|
using CODLayerT =
|
|
orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>;
|
|
|
|
using CallbackManagerBuilder =
|
|
std::function<std::unique_ptr<CompileCallbackMgr>()>;
|
|
|
|
using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
|
|
|
|
private:
|
|
|
|
using OwningObject = object::OwningBinary<object::ObjectFile>;
|
|
|
|
class CBindingsResolver : public orc::SymbolResolver {
|
|
public:
|
|
CBindingsResolver(OrcCBindingsStack &Stack,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx)
|
|
: Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
|
|
ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
|
|
|
|
orc::SymbolNameSet
|
|
getResponsibilitySet(const orc::SymbolNameSet &Symbols) override {
|
|
orc::SymbolNameSet Result;
|
|
|
|
for (auto &S : Symbols) {
|
|
if (auto Sym = findSymbol(*S)) {
|
|
if (!Sym.getFlags().isStrong())
|
|
Result.insert(S);
|
|
} else if (auto Err = Sym.takeError()) {
|
|
Stack.reportError(std::move(Err));
|
|
return orc::SymbolNameSet();
|
|
}
|
|
}
|
|
|
|
return Result;
|
|
}
|
|
|
|
orc::SymbolNameSet
|
|
lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
|
|
orc::SymbolNameSet Symbols) override {
|
|
orc::SymbolNameSet UnresolvedSymbols;
|
|
|
|
for (auto &S : Symbols) {
|
|
if (auto Sym = findSymbol(*S)) {
|
|
if (auto Addr = Sym.getAddress()) {
|
|
Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
|
|
Query->notifySymbolReady();
|
|
} else {
|
|
Stack.ES.legacyFailQuery(*Query, Addr.takeError());
|
|
return orc::SymbolNameSet();
|
|
}
|
|
} else if (auto Err = Sym.takeError()) {
|
|
Stack.ES.legacyFailQuery(*Query, std::move(Err));
|
|
return orc::SymbolNameSet();
|
|
} else
|
|
UnresolvedSymbols.insert(S);
|
|
}
|
|
|
|
if (Query->isFullyResolved())
|
|
Query->handleFullyResolved();
|
|
|
|
if (Query->isFullyReady())
|
|
Query->handleFullyReady();
|
|
|
|
return UnresolvedSymbols;
|
|
}
|
|
|
|
private:
|
|
JITSymbol findSymbol(const std::string &Name) {
|
|
// Search order:
|
|
// 1. JIT'd symbols.
|
|
// 2. Runtime overrides.
|
|
// 3. External resolver (if present).
|
|
|
|
if (Stack.CODLayer) {
|
|
if (auto Sym = Stack.CODLayer->findSymbol(Name, true))
|
|
return Sym;
|
|
else if (auto Err = Sym.takeError())
|
|
return Sym.takeError();
|
|
} else {
|
|
if (auto Sym = Stack.CompileLayer.findSymbol(Name, true))
|
|
return Sym;
|
|
else if (auto Err = Sym.takeError())
|
|
return Sym.takeError();
|
|
}
|
|
|
|
if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
|
|
return Sym;
|
|
|
|
if (ExternalResolver)
|
|
return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
|
|
JITSymbolFlags::Exported);
|
|
|
|
return JITSymbol(nullptr);
|
|
}
|
|
|
|
OrcCBindingsStack &Stack;
|
|
LLVMOrcSymbolResolverFn ExternalResolver;
|
|
void *ExternalResolverCtx = nullptr;
|
|
};
|
|
|
|
public:
|
|
OrcCBindingsStack(TargetMachine &TM,
|
|
IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
|
|
: CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()),
|
|
IndirectStubsMgr(IndirectStubsMgrBuilder()),
|
|
ObjectLayer(ES,
|
|
[this](orc::VModuleKey K) {
|
|
auto ResolverI = Resolvers.find(K);
|
|
assert(ResolverI != Resolvers.end() &&
|
|
"No resolver for module K");
|
|
auto Resolver = std::move(ResolverI->second);
|
|
Resolvers.erase(ResolverI);
|
|
return ObjLayerT::Resources{
|
|
std::make_shared<SectionMemoryManager>(), Resolver};
|
|
},
|
|
nullptr,
|
|
[this](orc::VModuleKey K, const object::ObjectFile &Obj,
|
|
const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
|
|
this->notifyFinalized(K, Obj, LoadedObjInfo);
|
|
},
|
|
[this](orc::VModuleKey K, const object::ObjectFile &Obj) {
|
|
this->notifyFreed(K, Obj);
|
|
}),
|
|
CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)),
|
|
CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(),
|
|
std::move(IndirectStubsMgrBuilder), Resolvers)),
|
|
CXXRuntimeOverrides(
|
|
[this](const std::string &S) { return mangle(S); }) {}
|
|
|
|
Error shutdown() {
|
|
// Run any destructors registered with __cxa_atexit.
|
|
CXXRuntimeOverrides.runDestructors();
|
|
// Run any IR destructors.
|
|
for (auto &DtorRunner : IRStaticDestructorRunners)
|
|
if (auto Err = DtorRunner.runViaLayer(*this))
|
|
return Err;
|
|
return Error::success();
|
|
}
|
|
|
|
std::string mangle(StringRef Name) {
|
|
std::string MangledName;
|
|
{
|
|
raw_string_ostream MangledNameStream(MangledName);
|
|
Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
|
|
}
|
|
return MangledName;
|
|
}
|
|
|
|
template <typename PtrTy>
|
|
static PtrTy fromTargetAddress(JITTargetAddress Addr) {
|
|
return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
|
|
}
|
|
|
|
Expected<JITTargetAddress>
|
|
createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,
|
|
void *CallbackCtx) {
|
|
auto WrappedCallback = [=]() -> JITTargetAddress {
|
|
return Callback(wrap(this), CallbackCtx);
|
|
};
|
|
|
|
return CCMgr->getCompileCallback(std::move(WrappedCallback));
|
|
}
|
|
|
|
Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) {
|
|
return IndirectStubsMgr->createStub(StubName, Addr,
|
|
JITSymbolFlags::Exported);
|
|
}
|
|
|
|
Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) {
|
|
return IndirectStubsMgr->updatePointer(Name, Addr);
|
|
}
|
|
|
|
template <typename LayerT>
|
|
Expected<orc::VModuleKey>
|
|
addIRModule(LayerT &Layer, std::unique_ptr<Module> M,
|
|
std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx) {
|
|
|
|
// Attach a data-layout if one isn't already present.
|
|
if (M->getDataLayout().isDefault())
|
|
M->setDataLayout(DL);
|
|
|
|
// Record the static constructors and destructors. We have to do this before
|
|
// we hand over ownership of the module to the JIT.
|
|
std::vector<std::string> CtorNames, DtorNames;
|
|
for (auto Ctor : orc::getConstructors(*M))
|
|
CtorNames.push_back(mangle(Ctor.Func->getName()));
|
|
for (auto Dtor : orc::getDestructors(*M))
|
|
DtorNames.push_back(mangle(Dtor.Func->getName()));
|
|
|
|
// Add the module to the JIT.
|
|
auto K = ES.allocateVModule();
|
|
Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver,
|
|
ExternalResolverCtx);
|
|
if (auto Err = Layer.addModule(K, std::move(M)))
|
|
return std::move(Err);
|
|
|
|
KeyLayers[K] = detail::createGenericLayer(Layer);
|
|
|
|
// Run the static constructors, and save the static destructor runner for
|
|
// execution when the JIT is torn down.
|
|
orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), K);
|
|
if (auto Err = CtorRunner.runViaLayer(*this))
|
|
return std::move(Err);
|
|
|
|
IRStaticDestructorRunners.emplace_back(std::move(DtorNames), K);
|
|
|
|
return K;
|
|
}
|
|
|
|
Expected<orc::VModuleKey>
|
|
addIRModuleEager(std::unique_ptr<Module> M,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx) {
|
|
return addIRModule(CompileLayer, std::move(M),
|
|
llvm::make_unique<SectionMemoryManager>(),
|
|
std::move(ExternalResolver), ExternalResolverCtx);
|
|
}
|
|
|
|
Expected<orc::VModuleKey>
|
|
addIRModuleLazy(std::unique_ptr<Module> M,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx) {
|
|
if (!CODLayer)
|
|
return make_error<StringError>("Can not add lazy module: No compile "
|
|
"callback manager available",
|
|
inconvertibleErrorCode());
|
|
|
|
return addIRModule(*CODLayer, std::move(M),
|
|
llvm::make_unique<SectionMemoryManager>(),
|
|
std::move(ExternalResolver), ExternalResolverCtx);
|
|
}
|
|
|
|
Error removeModule(orc::VModuleKey K) {
|
|
// FIXME: Should error release the module key?
|
|
if (auto Err = KeyLayers[K]->removeModule(K))
|
|
return Err;
|
|
ES.releaseVModule(K);
|
|
KeyLayers.erase(K);
|
|
return Error::success();
|
|
}
|
|
|
|
Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,
|
|
LLVMOrcSymbolResolverFn ExternalResolver,
|
|
void *ExternalResolverCtx) {
|
|
if (auto Obj = object::ObjectFile::createObjectFile(
|
|
ObjBuffer->getMemBufferRef())) {
|
|
|
|
auto K = ES.allocateVModule();
|
|
Resolvers[K] = std::make_shared<CBindingsResolver>(
|
|
*this, ExternalResolver, ExternalResolverCtx);
|
|
|
|
if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer)))
|
|
return std::move(Err);
|
|
|
|
KeyLayers[K] = detail::createGenericLayer(ObjectLayer);
|
|
|
|
return K;
|
|
} else
|
|
return Obj.takeError();
|
|
}
|
|
|
|
JITSymbol findSymbol(const std::string &Name,
|
|
bool ExportedSymbolsOnly) {
|
|
if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly))
|
|
return Sym;
|
|
if (CODLayer)
|
|
return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly);
|
|
return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
|
|
}
|
|
|
|
JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
|
|
bool ExportedSymbolsOnly) {
|
|
assert(KeyLayers.count(K) && "looking up symbol in unknown module");
|
|
return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
|
|
}
|
|
|
|
Expected<JITTargetAddress> findSymbolAddress(const std::string &Name,
|
|
bool ExportedSymbolsOnly) {
|
|
if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) {
|
|
// Successful lookup, non-null symbol:
|
|
if (auto AddrOrErr = Sym.getAddress())
|
|
return *AddrOrErr;
|
|
else
|
|
return AddrOrErr.takeError();
|
|
} else if (auto Err = Sym.takeError()) {
|
|
// Lookup failure - report error.
|
|
return std::move(Err);
|
|
}
|
|
|
|
// No symbol not found. Return 0.
|
|
return 0;
|
|
}
|
|
|
|
Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K,
|
|
const std::string &Name,
|
|
bool ExportedSymbolsOnly) {
|
|
if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
|
|
// Successful lookup, non-null symbol:
|
|
if (auto AddrOrErr = Sym.getAddress())
|
|
return *AddrOrErr;
|
|
else
|
|
return AddrOrErr.takeError();
|
|
} else if (auto Err = Sym.takeError()) {
|
|
// Lookup failure - report error.
|
|
return std::move(Err);
|
|
}
|
|
|
|
// Symbol not found. Return 0.
|
|
return 0;
|
|
}
|
|
|
|
const std::string &getErrorMessage() const { return ErrMsg; }
|
|
|
|
void RegisterJITEventListener(JITEventListener *L) {
|
|
if (!L)
|
|
return;
|
|
EventListeners.push_back(L);
|
|
}
|
|
|
|
void UnregisterJITEventListener(JITEventListener *L) {
|
|
if (!L)
|
|
return;
|
|
|
|
auto I = find(reverse(EventListeners), L);
|
|
if (I != EventListeners.rend()) {
|
|
std::swap(*I, EventListeners.back());
|
|
EventListeners.pop_back();
|
|
}
|
|
}
|
|
|
|
private:
|
|
using ResolverMap =
|
|
std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>;
|
|
|
|
static std::unique_ptr<CompileCallbackMgr>
|
|
createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) {
|
|
auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0);
|
|
if (!CCMgr) {
|
|
// FIXME: It would be good if we could report this somewhere, but we do
|
|
// have an instance yet.
|
|
logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: ");
|
|
return nullptr;
|
|
}
|
|
return std::move(*CCMgr);
|
|
}
|
|
|
|
static std::unique_ptr<CODLayerT>
|
|
createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer,
|
|
CompileCallbackMgr *CCMgr,
|
|
IndirectStubsManagerBuilder IndirectStubsMgrBuilder,
|
|
ResolverMap &Resolvers) {
|
|
// If there is no compile callback manager available we can not create a
|
|
// compile on demand layer.
|
|
if (!CCMgr)
|
|
return nullptr;
|
|
|
|
return llvm::make_unique<CODLayerT>(
|
|
ES, CompileLayer,
|
|
[&Resolvers](orc::VModuleKey K) {
|
|
auto ResolverI = Resolvers.find(K);
|
|
assert(ResolverI != Resolvers.end() && "No resolver for module K");
|
|
return ResolverI->second;
|
|
},
|
|
[&Resolvers](orc::VModuleKey K,
|
|
std::shared_ptr<orc::SymbolResolver> Resolver) {
|
|
assert(!Resolvers.count(K) && "Resolver already present");
|
|
Resolvers[K] = std::move(Resolver);
|
|
},
|
|
[](Function &F) { return std::set<Function *>({&F}); }, *CCMgr,
|
|
std::move(IndirectStubsMgrBuilder), false);
|
|
}
|
|
|
|
void reportError(Error Err) {
|
|
// FIXME: Report errors on the execution session.
|
|
logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
|
|
};
|
|
|
|
void notifyFinalized(orc::VModuleKey K,
|
|
const object::ObjectFile &Obj,
|
|
const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
|
|
uint64_t Key = static_cast<uint64_t>(
|
|
reinterpret_cast<uintptr_t>(Obj.getData().data()));
|
|
for (auto &Listener : EventListeners)
|
|
Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo);
|
|
}
|
|
|
|
void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
|
|
uint64_t Key = static_cast<uint64_t>(
|
|
reinterpret_cast<uintptr_t>(Obj.getData().data()));
|
|
for (auto &Listener : EventListeners)
|
|
Listener->notifyFreeingObject(Key);
|
|
}
|
|
|
|
orc::ExecutionSession ES;
|
|
std::unique_ptr<CompileCallbackMgr> CCMgr;
|
|
|
|
std::vector<JITEventListener *> EventListeners;
|
|
|
|
DataLayout DL;
|
|
SectionMemoryManager CCMgrMemMgr;
|
|
|
|
std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
|
|
|
|
ObjLayerT ObjectLayer;
|
|
CompileLayerT CompileLayer;
|
|
std::unique_ptr<CODLayerT> CODLayer;
|
|
|
|
std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
|
|
|
|
orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides;
|
|
std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
|
|
std::string ErrMsg;
|
|
|
|
ResolverMap Resolvers;
|
|
};
|
|
|
|
} // end namespace llvm
|
|
|
|
#endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
|