3b5842c9c4
Part of ongoing work to make core ORC operations asynchronous. This only affects the runner utilities, not the AllocationActions themselves. The AllocationActions will be made asynchronous in a future patch.
383 lines
12 KiB
C++
383 lines
12 KiB
C++
//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
|
|
#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
|
|
#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
|
|
#include "llvm/Support/MSVCErrorWorkarounds.h"
|
|
#include "llvm/Support/Process.h"
|
|
#include "llvm/Support/WindowsError.h"
|
|
#include <future>
|
|
#include <sstream>
|
|
|
|
#if defined(LLVM_ON_UNIX)
|
|
#include <errno.h>
|
|
#include <fcntl.h>
|
|
#include <sys/mman.h>
|
|
#if defined(__MVS__)
|
|
#include "llvm/Support/BLAKE3.h"
|
|
#include <sys/shm.h>
|
|
#endif
|
|
#include <unistd.h>
|
|
#endif
|
|
|
|
namespace llvm {
|
|
namespace orc {
|
|
namespace rt_bootstrap {
|
|
|
|
#if defined(_WIN32)
|
|
static DWORD getWindowsProtectionFlags(MemProt MP) {
|
|
if (MP == MemProt::Read)
|
|
return PAGE_READONLY;
|
|
if (MP == MemProt::Write ||
|
|
MP == (MemProt::Write | MemProt::Read)) {
|
|
// Note: PAGE_WRITE is not supported by VirtualProtect
|
|
return PAGE_READWRITE;
|
|
}
|
|
if (MP == (MemProt::Read | MemProt::Exec))
|
|
return PAGE_EXECUTE_READ;
|
|
if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))
|
|
return PAGE_EXECUTE_READWRITE;
|
|
if (MP == MemProt::Exec)
|
|
return PAGE_EXECUTE;
|
|
|
|
return PAGE_NOACCESS;
|
|
}
|
|
#endif
|
|
|
|
Expected<std::pair<ExecutorAddr, std::string>>
|
|
ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
|
|
#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
|
|
|
|
#if defined(LLVM_ON_UNIX)
|
|
|
|
std::string SharedMemoryName;
|
|
{
|
|
std::stringstream SharedMemoryNameStream;
|
|
SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
|
|
<< (++SharedMemoryCount);
|
|
SharedMemoryName = SharedMemoryNameStream.str();
|
|
}
|
|
|
|
#if defined(__MVS__)
|
|
ArrayRef<uint8_t> Data(
|
|
reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
|
|
SharedMemoryName.size());
|
|
auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
|
|
key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
|
|
int SharedMemoryId =
|
|
shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);
|
|
if (SharedMemoryId < 0)
|
|
return errorCodeToError(errnoAsErrorCode());
|
|
|
|
void *Addr = shmat(SharedMemoryId, nullptr, 0);
|
|
if (Addr == reinterpret_cast<void *>(-1))
|
|
return errorCodeToError(errnoAsErrorCode());
|
|
#else
|
|
int SharedMemoryFile =
|
|
shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700);
|
|
if (SharedMemoryFile < 0)
|
|
return errorCodeToError(errnoAsErrorCode());
|
|
|
|
// by default size is 0
|
|
if (ftruncate(SharedMemoryFile, Size) < 0)
|
|
return errorCodeToError(errnoAsErrorCode());
|
|
|
|
void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0);
|
|
if (Addr == MAP_FAILED)
|
|
return errorCodeToError(errnoAsErrorCode());
|
|
|
|
close(SharedMemoryFile);
|
|
#endif
|
|
|
|
#elif defined(_WIN32)
|
|
|
|
std::string SharedMemoryName;
|
|
{
|
|
std::stringstream SharedMemoryNameStream;
|
|
SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
|
|
<< (++SharedMemoryCount);
|
|
SharedMemoryName = SharedMemoryNameStream.str();
|
|
}
|
|
|
|
std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
|
|
SharedMemoryName.end());
|
|
HANDLE SharedMemoryFile = CreateFileMappingW(
|
|
INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
|
|
Size & 0xffffffff, WideSharedMemoryName.c_str());
|
|
if (!SharedMemoryFile)
|
|
return errorCodeToError(mapWindowsError(GetLastError()));
|
|
|
|
void *Addr = MapViewOfFile(SharedMemoryFile,
|
|
FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
|
|
if (!Addr) {
|
|
CloseHandle(SharedMemoryFile);
|
|
return errorCodeToError(mapWindowsError(GetLastError()));
|
|
}
|
|
|
|
#endif
|
|
|
|
{
|
|
std::lock_guard<std::mutex> Lock(Mutex);
|
|
Reservations[Addr].Size = Size;
|
|
#if defined(_WIN32)
|
|
Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
|
|
#endif
|
|
}
|
|
|
|
return std::make_pair(ExecutorAddr::fromPtr(Addr),
|
|
std::move(SharedMemoryName));
|
|
#else
|
|
return make_error<StringError>(
|
|
"SharedMemoryMapper is not supported on this platform yet",
|
|
inconvertibleErrorCode());
|
|
#endif
|
|
}
|
|
|
|
Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(
|
|
ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {
|
|
#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
|
|
|
|
ExecutorAddr MinAddr(~0ULL);
|
|
|
|
// Contents are already in place
|
|
for (auto &Segment : FR.Segments) {
|
|
if (Segment.Addr < MinAddr)
|
|
MinAddr = Segment.Addr;
|
|
|
|
#if defined(LLVM_ON_UNIX)
|
|
|
|
#if defined(__MVS__)
|
|
// TODO Is it possible to change the protection level?
|
|
#else
|
|
int NativeProt = 0;
|
|
if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)
|
|
NativeProt |= PROT_READ;
|
|
if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)
|
|
NativeProt |= PROT_WRITE;
|
|
if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
|
|
NativeProt |= PROT_EXEC;
|
|
|
|
if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt))
|
|
return errorCodeToError(errnoAsErrorCode());
|
|
#endif
|
|
|
|
#elif defined(_WIN32)
|
|
|
|
DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);
|
|
|
|
if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
|
|
&NativeProt))
|
|
return errorCodeToError(mapWindowsError(GetLastError()));
|
|
|
|
#endif
|
|
|
|
if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)
|
|
sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(),
|
|
Segment.Size);
|
|
}
|
|
|
|
// Run finalization actions and get deinitlization action list.
|
|
std::vector<shared::WrapperFunctionCall> DeinitializeActions;
|
|
{
|
|
std::promise<MSVCPExpected<std::vector<shared::WrapperFunctionCall>>> P;
|
|
auto F = P.get_future();
|
|
shared::runFinalizeActions(
|
|
FR.Actions, [&](Expected<std::vector<shared::WrapperFunctionCall>> R) {
|
|
P.set_value(std::move(R));
|
|
});
|
|
if (auto DeinitializeActionsOrErr = F.get())
|
|
DeinitializeActions = std::move(*DeinitializeActionsOrErr);
|
|
else
|
|
return DeinitializeActionsOrErr.takeError();
|
|
}
|
|
|
|
{
|
|
std::lock_guard<std::mutex> Lock(Mutex);
|
|
Allocations[MinAddr].DeinitializationActions =
|
|
std::move(DeinitializeActions);
|
|
Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr);
|
|
}
|
|
|
|
return MinAddr;
|
|
|
|
#else
|
|
return make_error<StringError>(
|
|
"SharedMemoryMapper is not supported on this platform yet",
|
|
inconvertibleErrorCode());
|
|
#endif
|
|
}
|
|
|
|
Error ExecutorSharedMemoryMapperService::deinitialize(
|
|
const std::vector<ExecutorAddr> &Bases) {
|
|
Error AllErr = Error::success();
|
|
|
|
{
|
|
std::lock_guard<std::mutex> Lock(Mutex);
|
|
|
|
for (auto Base : llvm::reverse(Bases)) {
|
|
shared::runDeallocActions(
|
|
Allocations[Base].DeinitializationActions, [&](Error Err) {
|
|
if (Err)
|
|
AllErr = joinErrors(std::move(AllErr), std::move(Err));
|
|
});
|
|
|
|
// Remove the allocation from the allocation list of its reservation
|
|
for (auto &Reservation : Reservations) {
|
|
auto AllocationIt = llvm::find(Reservation.second.Allocations, Base);
|
|
if (AllocationIt != Reservation.second.Allocations.end()) {
|
|
Reservation.second.Allocations.erase(AllocationIt);
|
|
break;
|
|
}
|
|
}
|
|
|
|
Allocations.erase(Base);
|
|
}
|
|
}
|
|
|
|
return AllErr;
|
|
}
|
|
|
|
Error ExecutorSharedMemoryMapperService::release(
|
|
const std::vector<ExecutorAddr> &Bases) {
|
|
#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
|
|
Error Err = Error::success();
|
|
|
|
for (auto Base : Bases) {
|
|
std::vector<ExecutorAddr> AllocAddrs;
|
|
size_t Size;
|
|
|
|
#if defined(_WIN32)
|
|
HANDLE SharedMemoryFile;
|
|
#endif
|
|
|
|
{
|
|
std::lock_guard<std::mutex> Lock(Mutex);
|
|
auto &R = Reservations[Base.toPtr<void *>()];
|
|
Size = R.Size;
|
|
|
|
#if defined(_WIN32)
|
|
SharedMemoryFile = R.SharedMemoryFile;
|
|
#endif
|
|
|
|
AllocAddrs.swap(R.Allocations);
|
|
}
|
|
|
|
// deinitialize sub allocations
|
|
if (Error E = deinitialize(AllocAddrs))
|
|
Err = joinErrors(std::move(Err), std::move(E));
|
|
|
|
#if defined(LLVM_ON_UNIX)
|
|
|
|
#if defined(__MVS__)
|
|
(void)Size;
|
|
|
|
if (shmdt(Base.toPtr<void *>()) < 0)
|
|
Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
|
|
#else
|
|
if (munmap(Base.toPtr<void *>(), Size) != 0)
|
|
Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
|
|
#endif
|
|
|
|
#elif defined(_WIN32)
|
|
(void)Size;
|
|
|
|
if (!UnmapViewOfFile(Base.toPtr<void *>()))
|
|
Err = joinErrors(std::move(Err),
|
|
errorCodeToError(mapWindowsError(GetLastError())));
|
|
|
|
CloseHandle(SharedMemoryFile);
|
|
|
|
#endif
|
|
|
|
std::lock_guard<std::mutex> Lock(Mutex);
|
|
Reservations.erase(Base.toPtr<void *>());
|
|
}
|
|
|
|
return Err;
|
|
#else
|
|
return make_error<StringError>(
|
|
"SharedMemoryMapper is not supported on this platform yet",
|
|
inconvertibleErrorCode());
|
|
#endif
|
|
}
|
|
|
|
Error ExecutorSharedMemoryMapperService::shutdown() {
|
|
if (Reservations.empty())
|
|
return Error::success();
|
|
|
|
std::vector<ExecutorAddr> ReservationAddrs;
|
|
ReservationAddrs.reserve(Reservations.size());
|
|
for (const auto &R : Reservations)
|
|
ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
|
|
|
|
return release(std::move(ReservationAddrs));
|
|
}
|
|
|
|
void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
|
|
StringMap<ExecutorAddr> &M) {
|
|
M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
|
|
ExecutorAddr::fromPtr(this);
|
|
M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
|
|
ExecutorAddr::fromPtr(&reserveWrapper);
|
|
M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
|
|
ExecutorAddr::fromPtr(&initializeWrapper);
|
|
M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
|
|
ExecutorAddr::fromPtr(&deinitializeWrapper);
|
|
M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
|
|
ExecutorAddr::fromPtr(&releaseWrapper);
|
|
}
|
|
|
|
llvm::orc::shared::CWrapperFunctionResult
|
|
ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
|
|
size_t ArgSize) {
|
|
return shared::WrapperFunction<
|
|
rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
|
|
handle(ArgData, ArgSize,
|
|
shared::makeMethodWrapperHandler(
|
|
&ExecutorSharedMemoryMapperService::reserve))
|
|
.release();
|
|
}
|
|
|
|
llvm::orc::shared::CWrapperFunctionResult
|
|
ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
|
|
size_t ArgSize) {
|
|
return shared::WrapperFunction<
|
|
rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
|
|
handle(ArgData, ArgSize,
|
|
shared::makeMethodWrapperHandler(
|
|
&ExecutorSharedMemoryMapperService::initialize))
|
|
.release();
|
|
}
|
|
|
|
llvm::orc::shared::CWrapperFunctionResult
|
|
ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
|
|
size_t ArgSize) {
|
|
return shared::WrapperFunction<
|
|
rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
|
|
handle(ArgData, ArgSize,
|
|
shared::makeMethodWrapperHandler(
|
|
&ExecutorSharedMemoryMapperService::deinitialize))
|
|
.release();
|
|
}
|
|
|
|
llvm::orc::shared::CWrapperFunctionResult
|
|
ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
|
|
size_t ArgSize) {
|
|
return shared::WrapperFunction<
|
|
rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
|
|
handle(ArgData, ArgSize,
|
|
shared::makeMethodWrapperHandler(
|
|
&ExecutorSharedMemoryMapperService::release))
|
|
.release();
|
|
}
|
|
|
|
} // namespace rt_bootstrap
|
|
} // end namespace orc
|
|
} // end namespace llvm
|