9ba54ca3ee
On Windows, for a reason I don't fully understand boolean bits get extra padding (even when asking for packed structures) in the structures that messes the offsets between the compiler and the runtime. Also, "weak" works differently on Windows than Linux (i.e., the "local" routine has preference) which causes it to crash as we don't really have an alternate implementation of __kmpc_omp_wait_deps. Given this, it doesn't make sense to mark it as "weak" for Linux either.
504 lines
17 KiB
C++
504 lines
17 KiB
C++
//===-- InteropAPI.cpp - Implementation of OpenMP interoperability API ----===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "OpenMP/InteropAPI.h"
|
|
#include "OpenMP/InternalTypes.h"
|
|
#include "OpenMP/omp.h"
|
|
|
|
#include "OffloadPolicy.h"
|
|
#include "PluginManager.h"
|
|
#include "device.h"
|
|
#include "omptarget.h"
|
|
#include "llvm/Support/Error.h"
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
|
|
extern "C" {
|
|
|
|
void __kmpc_omp_wait_deps(ident_t *loc_ref, int32_t gtid, int32_t ndeps,
|
|
kmp_depend_info_t *dep_list, int32_t ndeps_noalias,
|
|
kmp_depend_info_t *noalias_dep_list);
|
|
|
|
} // extern "C"
|
|
|
|
namespace {
|
|
omp_interop_rc_t getPropertyErrorType(omp_interop_property_t Property) {
|
|
switch (Property) {
|
|
case omp_ipr_fr_id:
|
|
return omp_irc_type_int;
|
|
case omp_ipr_fr_name:
|
|
return omp_irc_type_str;
|
|
case omp_ipr_vendor:
|
|
return omp_irc_type_int;
|
|
case omp_ipr_vendor_name:
|
|
return omp_irc_type_str;
|
|
case omp_ipr_device_num:
|
|
return omp_irc_type_int;
|
|
case omp_ipr_platform:
|
|
return omp_irc_type_int;
|
|
case omp_ipr_device:
|
|
return omp_irc_type_ptr;
|
|
case omp_ipr_device_context:
|
|
return omp_irc_type_ptr;
|
|
case omp_ipr_targetsync:
|
|
return omp_irc_type_ptr;
|
|
};
|
|
return omp_irc_no_value;
|
|
}
|
|
|
|
void getTypeMismatch(omp_interop_property_t Property, int *Err) {
|
|
if (Err)
|
|
*Err = getPropertyErrorType(Property);
|
|
}
|
|
|
|
static const char *VendorStrTbl[] = {
|
|
"unknown", "amd", "arm", "bsc", "fujitsu", "gnu", "hpe",
|
|
"ibm", "intel", "llvm", "nec", "nvidia", "ti"};
|
|
const char *getVendorIdToStr(const omp_vendor_id_t VendorId) {
|
|
if (VendorId < omp_vendor_unknown || VendorId >= omp_vendor_last)
|
|
return ("unknown");
|
|
return VendorStrTbl[VendorId];
|
|
}
|
|
|
|
static const char *ForeignRuntimeStrTbl[] = {
|
|
"none", "cuda", "cuda_driver", "opencl",
|
|
"sycl", "hip", "level_zero", "hsa"};
|
|
const char *getForeignRuntimeIdToStr(const tgt_foreign_runtime_id_t FrId) {
|
|
if (FrId < tgt_fr_none || FrId >= tgt_fr_last)
|
|
return ("unknown");
|
|
return ForeignRuntimeStrTbl[FrId];
|
|
}
|
|
|
|
template <typename PropertyTy>
|
|
PropertyTy getProperty(omp_interop_val_t &InteropVal,
|
|
omp_interop_property_t Property, int *Err);
|
|
|
|
template <>
|
|
intptr_t getProperty<intptr_t>(omp_interop_val_t &InteropVal,
|
|
omp_interop_property_t Property, int *Err) {
|
|
switch (Property) {
|
|
case omp_ipr_fr_id:
|
|
return InteropVal.fr_id;
|
|
case omp_ipr_vendor:
|
|
return InteropVal.vendor_id;
|
|
case omp_ipr_device_num:
|
|
return InteropVal.device_id;
|
|
default:;
|
|
}
|
|
getTypeMismatch(Property, Err);
|
|
return 0;
|
|
}
|
|
|
|
template <>
|
|
const char *getProperty<const char *>(omp_interop_val_t &InteropVal,
|
|
omp_interop_property_t Property,
|
|
int *Err) {
|
|
switch (Property) {
|
|
case omp_ipr_fr_name:
|
|
return getForeignRuntimeIdToStr(InteropVal.fr_id);
|
|
case omp_ipr_vendor_name:
|
|
return getVendorIdToStr(InteropVal.vendor_id);
|
|
default:
|
|
getTypeMismatch(Property, Err);
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
template <>
|
|
void *getProperty<void *>(omp_interop_val_t &InteropVal,
|
|
omp_interop_property_t Property, int *Err) {
|
|
switch (Property) {
|
|
case omp_ipr_device:
|
|
if (InteropVal.device_info.Device)
|
|
return InteropVal.device_info.Device;
|
|
*Err = omp_irc_no_value;
|
|
return const_cast<char *>(InteropVal.err_str);
|
|
case omp_ipr_platform:
|
|
return InteropVal.device_info.Platform;
|
|
case omp_ipr_device_context:
|
|
return InteropVal.device_info.Context;
|
|
case omp_ipr_targetsync:
|
|
return InteropVal.async_info ? InteropVal.async_info->Queue : nullptr;
|
|
default:;
|
|
}
|
|
getTypeMismatch(Property, Err);
|
|
return nullptr;
|
|
}
|
|
|
|
bool getPropertyCheck(omp_interop_val_t **InteropPtr,
|
|
omp_interop_property_t Property, int *Err) {
|
|
if (Err)
|
|
*Err = omp_irc_success;
|
|
if (!InteropPtr) {
|
|
if (Err)
|
|
*Err = omp_irc_empty;
|
|
return false;
|
|
}
|
|
if (Property >= 0 || Property < omp_ipr_first) {
|
|
if (Err)
|
|
*Err = omp_irc_out_of_range;
|
|
return false;
|
|
}
|
|
if (Property == omp_ipr_targetsync &&
|
|
(*InteropPtr)->interop_type != kmp_interop_type_targetsync) {
|
|
if (Err)
|
|
*Err = omp_irc_other;
|
|
return false;
|
|
}
|
|
if ((Property == omp_ipr_device || Property == omp_ipr_device_context) &&
|
|
(*InteropPtr)->interop_type == kmp_interop_type_targetsync) {
|
|
if (Err)
|
|
*Err = omp_irc_other;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
#define __OMP_GET_INTEROP_TY(RETURN_TYPE, SUFFIX) \
|
|
RETURN_TYPE omp_get_interop_##SUFFIX(const omp_interop_t interop, \
|
|
omp_interop_property_t property_id, \
|
|
int *err) { \
|
|
omp_interop_val_t *interop_val = (omp_interop_val_t *)interop; \
|
|
if (!getPropertyCheck(&interop_val, property_id, err)) { \
|
|
return (RETURN_TYPE)(0); \
|
|
} \
|
|
return getProperty<RETURN_TYPE>(*interop_val, property_id, err); \
|
|
}
|
|
__OMP_GET_INTEROP_TY(intptr_t, int)
|
|
__OMP_GET_INTEROP_TY(void *, ptr)
|
|
__OMP_GET_INTEROP_TY(const char *, str)
|
|
#undef __OMP_GET_INTEROP_TY
|
|
|
|
#define __OMP_GET_INTEROP_TY3(RETURN_TYPE, SUFFIX) \
|
|
RETURN_TYPE omp_get_interop_##SUFFIX(const omp_interop_t interop, \
|
|
omp_interop_property_t property_id) { \
|
|
int err; \
|
|
omp_interop_val_t *interop_val = (omp_interop_val_t *)interop; \
|
|
if (!getPropertyCheck(&interop_val, property_id, &err)) { \
|
|
return (RETURN_TYPE)(0); \
|
|
} \
|
|
return nullptr; \
|
|
return getProperty<RETURN_TYPE>(*interop_val, property_id, &err); \
|
|
}
|
|
__OMP_GET_INTEROP_TY3(const char *, name)
|
|
__OMP_GET_INTEROP_TY3(const char *, type_desc)
|
|
__OMP_GET_INTEROP_TY3(const char *, rc_desc)
|
|
#undef __OMP_GET_INTEROP_TY3
|
|
|
|
extern "C" {
|
|
|
|
omp_interop_val_t *__tgt_interop_get(ident_t *LocRef, int32_t InteropType,
|
|
int64_t DeviceNum, int32_t NumPrefers,
|
|
interop_spec_t *Prefers,
|
|
interop_ctx_t *Ctx, dep_pack_t *Deps) {
|
|
|
|
DP("Call to %s with device_num %" PRId64 ", interop type %" PRId32
|
|
", number of preferred specs %" PRId32 "%s%s\n",
|
|
__func__, DeviceNum, InteropType, NumPrefers,
|
|
Ctx->flags.implicit ? " (implicit)" : "",
|
|
Ctx->flags.nowait ? " (nowait)" : "");
|
|
|
|
if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED)
|
|
return omp_interop_none;
|
|
|
|
// Now, try to create an interop with device_num.
|
|
if (DeviceNum == OFFLOAD_DEVICE_DEFAULT)
|
|
DeviceNum = omp_get_default_device();
|
|
|
|
auto gtid = Ctx->gtid;
|
|
|
|
if (InteropType == kmp_interop_type_targetsync) {
|
|
if (Ctx->flags.nowait)
|
|
DP("Warning: nowait flag on interop creation not supported yet. "
|
|
"Ignored\n");
|
|
if (Deps)
|
|
__kmpc_omp_wait_deps(LocRef, gtid, Deps->ndeps, Deps->deplist,
|
|
Deps->ndeps_noalias, Deps->noalias_deplist);
|
|
}
|
|
|
|
auto DeviceOrErr = PM->getDevice(DeviceNum);
|
|
if (!DeviceOrErr) {
|
|
DP("Couldn't find device %" PRId64
|
|
" while constructing interop object: %s\n",
|
|
DeviceNum, toString(DeviceOrErr.takeError()).c_str());
|
|
return omp_interop_none;
|
|
}
|
|
auto &Device = *DeviceOrErr;
|
|
omp_interop_val_t *Interop = omp_interop_none;
|
|
auto InteropSpec = Device.RTL->select_interop_preference(
|
|
DeviceNum, InteropType, NumPrefers, Prefers);
|
|
if (InteropSpec.fr_id == tgt_fr_none) {
|
|
DP("Interop request not supported by device %" PRId64 "\n", DeviceNum);
|
|
return omp_interop_none;
|
|
}
|
|
DP("Selected interop preference is fr_id=%s%s impl_attrs=%" PRId64 "\n",
|
|
getForeignRuntimeIdToStr((tgt_foreign_runtime_id_t)InteropSpec.fr_id),
|
|
InteropSpec.attrs.inorder ? " inorder" : "", InteropSpec.impl_attrs);
|
|
|
|
if (Ctx->flags.implicit) {
|
|
// This is a request for an RTL managed interop object.
|
|
// Get it from the InteropTbl if possible
|
|
for (auto iop : PM->InteropTbl) {
|
|
if (iop->isCompatibleWith(InteropType, InteropSpec, DeviceNum, gtid)) {
|
|
Interop = iop;
|
|
Interop->markDirty();
|
|
DP("Reused interop " DPxMOD " from device number %" PRId64
|
|
" for gtid %" PRId32 "\n",
|
|
DPxPTR(Interop), DeviceNum, gtid);
|
|
return Interop;
|
|
}
|
|
}
|
|
}
|
|
|
|
Interop = Device.RTL->create_interop(DeviceNum, InteropType, &InteropSpec);
|
|
DP("Created an interop " DPxMOD " from device number %" PRId64 "\n",
|
|
DPxPTR(Interop), DeviceNum);
|
|
|
|
if (Ctx->flags.implicit) {
|
|
// register the new implicit interop in the RTL
|
|
Interop->setOwner(gtid);
|
|
Interop->markDirty();
|
|
PM->InteropTbl.add(Interop);
|
|
} else {
|
|
Interop->setOwner(omp_interop_val_t::no_owner);
|
|
}
|
|
|
|
return Interop;
|
|
}
|
|
|
|
int __tgt_interop_use60(ident_t *LocRef, omp_interop_val_t *Interop,
|
|
interop_ctx_t *Ctx, dep_pack_t *Deps) {
|
|
bool Nowait = Ctx->flags.nowait;
|
|
DP("Call to %s with interop " DPxMOD ", nowait %" PRId32 "\n", __func__,
|
|
DPxPTR(Interop), Nowait);
|
|
if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)
|
|
return OFFLOAD_FAIL;
|
|
|
|
if (Interop->interop_type == kmp_interop_type_targetsync) {
|
|
if (Deps) {
|
|
if (Nowait) {
|
|
DP("Warning: nowait flag on interop use with dependences not supported"
|
|
"yet. Ignored\n");
|
|
Nowait = false;
|
|
}
|
|
|
|
__kmpc_omp_wait_deps(LocRef, Ctx->gtid, Deps->ndeps, Deps->deplist,
|
|
Deps->ndeps_noalias, Deps->noalias_deplist);
|
|
}
|
|
}
|
|
|
|
auto DeviceOrErr = Interop->getDevice();
|
|
if (!DeviceOrErr) {
|
|
REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(Interop),
|
|
toString(DeviceOrErr.takeError()).c_str());
|
|
return OFFLOAD_FAIL;
|
|
}
|
|
auto &IOPDevice = *DeviceOrErr;
|
|
|
|
if (Interop->async_info && Interop->async_info->Queue) {
|
|
if (Nowait)
|
|
Interop->async_barrier(IOPDevice);
|
|
else {
|
|
Interop->flush(IOPDevice);
|
|
Interop->sync_barrier(IOPDevice);
|
|
Interop->markClean();
|
|
}
|
|
}
|
|
|
|
return OFFLOAD_SUCCESS;
|
|
}
|
|
|
|
int __tgt_interop_release(ident_t *LocRef, omp_interop_val_t *Interop,
|
|
interop_ctx_t *Ctx, dep_pack_t *Deps) {
|
|
DP("Call to %s with interop " DPxMOD "\n", __func__, DPxPTR(Interop));
|
|
|
|
if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)
|
|
return OFFLOAD_FAIL;
|
|
|
|
if (Interop->interop_type == kmp_interop_type_targetsync) {
|
|
if (Ctx->flags.nowait)
|
|
DP("Warning: nowait flag on interop destroy not supported "
|
|
"yet. Ignored\n");
|
|
if (Deps) {
|
|
__kmpc_omp_wait_deps(LocRef, Ctx->gtid, Deps->ndeps, Deps->deplist,
|
|
Deps->ndeps_noalias, Deps->noalias_deplist);
|
|
}
|
|
}
|
|
|
|
auto DeviceOrErr = Interop->getDevice();
|
|
if (!DeviceOrErr) {
|
|
REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(Interop),
|
|
toString(DeviceOrErr.takeError()).c_str());
|
|
return OFFLOAD_FAIL;
|
|
}
|
|
|
|
return Interop->release(*DeviceOrErr);
|
|
}
|
|
|
|
EXTERN int ompx_interop_add_completion_callback(omp_interop_val_t *Interop,
|
|
ompx_interop_cb_t *CB,
|
|
void *Data) {
|
|
DP("Call to %s with interop " DPxMOD ", property callback " DPxMOD
|
|
"and data " DPxMOD "\n",
|
|
__func__, DPxPTR(Interop), DPxPTR(CB), DPxPTR(Data));
|
|
|
|
if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)
|
|
return omp_irc_other;
|
|
|
|
Interop->addCompletionCb(CB, Data);
|
|
|
|
return omp_irc_success;
|
|
}
|
|
|
|
// Backwards compatibility wrappers
|
|
void __tgt_interop_init(ident_t *LocRef, int32_t Gtid,
|
|
omp_interop_val_t *&InteropPtr, int32_t InteropType,
|
|
int32_t DeviceId, int32_t Ndeps,
|
|
kmp_depend_info_t *DepList, int32_t HaveNowait) {
|
|
constexpr int32_t old_kmp_interop_type_targetsync = 2;
|
|
interop_ctx_t Ctx = {0, {false, (bool)HaveNowait, 0}, Gtid};
|
|
dep_pack_t Deps = {Ndeps, 0, DepList, nullptr};
|
|
InteropPtr =
|
|
__tgt_interop_get(LocRef,
|
|
InteropType == old_kmp_interop_type_targetsync
|
|
? kmp_interop_type_targetsync
|
|
: kmp_interop_type_target,
|
|
DeviceId, 0, nullptr, &Ctx, Ndeps ? &Deps : nullptr);
|
|
}
|
|
|
|
void __tgt_interop_use(ident_t *LocRef, int32_t Gtid,
|
|
omp_interop_val_t *&InteropPtr, int32_t DeviceId,
|
|
int32_t Ndeps, kmp_depend_info_t *DepList,
|
|
int32_t HaveNowait) {
|
|
interop_ctx_t Ctx = {0, {false, (bool)HaveNowait, 0}, Gtid};
|
|
dep_pack_t Deps = {Ndeps, 0, DepList, nullptr};
|
|
__tgt_interop_use60(LocRef, InteropPtr, &Ctx, Ndeps ? &Deps : nullptr);
|
|
}
|
|
|
|
void __tgt_interop_destroy(ident_t *LocRef, int32_t Gtid,
|
|
omp_interop_val_t *&InteropPtr, int32_t DeviceId,
|
|
int32_t Ndeps, kmp_depend_info_t *DepList,
|
|
int32_t HaveNowait) {
|
|
interop_ctx_t Ctx = {0, {false, (bool)HaveNowait, 0}, Gtid};
|
|
dep_pack_t Deps = {Ndeps, 0, DepList, nullptr};
|
|
__tgt_interop_release(LocRef, InteropPtr, &Ctx, Ndeps ? &Deps : nullptr);
|
|
}
|
|
|
|
} // extern "C"
|
|
|
|
llvm::Expected<DeviceTy &> omp_interop_val_t::getDevice() const {
|
|
return PM->getDevice(device_id);
|
|
}
|
|
|
|
bool omp_interop_val_t::isCompatibleWith(int32_t InteropType,
|
|
const interop_spec_t &Spec) {
|
|
if (interop_type != InteropType)
|
|
return false;
|
|
if (Spec.fr_id != fr_id)
|
|
return false;
|
|
if (Spec.attrs.inorder != attrs.inorder)
|
|
return false;
|
|
if (Spec.impl_attrs != impl_attrs)
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
bool omp_interop_val_t::isCompatibleWith(int32_t InteropType,
|
|
const interop_spec_t &Spec,
|
|
int64_t DeviceNum, int GTID) {
|
|
if (device_id != DeviceNum)
|
|
return false;
|
|
|
|
if (GTID != owner_gtid)
|
|
return false;
|
|
|
|
return isCompatibleWith(InteropType, Spec);
|
|
}
|
|
|
|
int32_t omp_interop_val_t::flush(DeviceTy &Device) {
|
|
return Device.RTL->flush_queue(this);
|
|
}
|
|
|
|
int32_t omp_interop_val_t::sync_barrier(DeviceTy &Device) {
|
|
if (Device.RTL->sync_barrier(this) != OFFLOAD_SUCCESS) {
|
|
FATAL_MESSAGE(device_id, "Interop sync barrier failed for %p object\n",
|
|
this);
|
|
}
|
|
DP("Calling completion callbacks for " DPxMOD "\n", DPxPTR(this));
|
|
runCompletionCbs();
|
|
return OFFLOAD_SUCCESS;
|
|
}
|
|
|
|
int32_t omp_interop_val_t::async_barrier(DeviceTy &Device) {
|
|
return Device.RTL->async_barrier(this);
|
|
}
|
|
|
|
int32_t omp_interop_val_t::release(DeviceTy &Device) {
|
|
if (async_info != nullptr && (!hasOwner() || !isClean())) {
|
|
flush(Device);
|
|
sync_barrier(Device);
|
|
}
|
|
return Device.RTL->release_interop(device_id, this);
|
|
}
|
|
|
|
void syncImplicitInterops(int Gtid, void *Event) {
|
|
if (PM->InteropTbl.size() == 0)
|
|
return;
|
|
|
|
DP("target_sync: syncing interops for gtid %" PRId32 ", event " DPxMOD "\n",
|
|
Gtid, DPxPTR(Event));
|
|
|
|
for (auto iop : PM->InteropTbl) {
|
|
if (iop->async_info && iop->async_info->Queue && iop->isOwnedBy(Gtid) &&
|
|
!iop->isClean()) {
|
|
|
|
auto DeviceOrErr = iop->getDevice();
|
|
if (!DeviceOrErr) {
|
|
REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(iop),
|
|
toString(DeviceOrErr.takeError()).c_str());
|
|
continue;
|
|
}
|
|
auto &IOPDevice = *DeviceOrErr;
|
|
|
|
iop->flush(IOPDevice);
|
|
iop->sync_barrier(IOPDevice);
|
|
iop->markClean();
|
|
|
|
// Alternate implementation option in case using barriers is not
|
|
// efficient enough:
|
|
//
|
|
// Instead of using a synchronous barrier, queue an asynchronous
|
|
// barrier and create a proxy task associated to the event to handle
|
|
// OpenMP synchronizations.
|
|
// When the event is completed, fulfill the proxy task to notify the
|
|
// OpenMP runtime.
|
|
// event = iop->asyncBarrier();
|
|
// ptask = createProxyTask();
|
|
// Events->add(event,ptask);
|
|
}
|
|
}
|
|
// This would be needed for the alternate implementation
|
|
// processEvents();
|
|
}
|
|
|
|
void InteropTblTy::clear() {
|
|
DP("Clearing Interop Table\n");
|
|
PerThreadTable::clear([](auto &IOP) {
|
|
auto DeviceOrErr = IOP->getDevice();
|
|
if (!DeviceOrErr) {
|
|
REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(IOP),
|
|
toString(DeviceOrErr.takeError()).c_str());
|
|
return;
|
|
}
|
|
IOP->release(*DeviceOrErr);
|
|
});
|
|
}
|