shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[aarch64][arm] Add support for the _Interlocked[Compare]ExchangePointer_{acq|nf|rel} MS intrinsics (#117645)

Browse Source 
Adds support for the following MSVC intrinsics:
* `_InterlockedCompareExchangePointer_acq`
* `_InterlockedCompareExchangePointer_rel`
* `_InterlockedExchangePointer_acq`
* `_InterlockedExchangePointer_nf`
* `_InterlockedExchangePointer_rel`

These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-170#interlocked-intrinsics>

NOTE: `_InterlockedCompareExchangePointer_nf` is not being added since
it already exists, although it was incorrectly added for all
architectures instead of being Arm & AArch64 specific.

This change also unifies how the pointer and non-pointer interlocked
compare-exchange intrinsics are being handled.
This commit is contained in:
Daniel Paoliello 2024-12-04 15:41:26 -06:00 committed by GitHub
parent a7da702377
commit 35c7df1a21
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 120 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
clang/include/clang/Basic/BuiltinsAArch64.def
View File

@ -182,6 +182,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@ -195,6 +198,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128, "UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128_acq,"UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")

5
clang/include/clang/Basic/BuiltinsARM.def
View File

@ -270,6 +270,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@ -283,6 +286,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_acq, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_nf, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")

65
clang/lib/CodeGen/CGBuiltin.cpp
View File

@ -522,8 +522,15 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
Address DestAddr = CheckAtomicAlignment(CGF, E);
auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
auto *RTy = Exchange->getType();
auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
if (RTy->isPointerTy()) {
Exchange = CGF.Builder.CreatePtrToInt(Exchange, CGF.IntPtrTy);
Comparand = CGF.Builder.CreatePtrToInt(Comparand, CGF.IntPtrTy);
}
// For Release ordering, the failure ordering should be Monotonic.
auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
@ -534,10 +541,16 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
// blocks the few atomics optimizations that LLVM has. If we want to optimize
// _Interlocked* operations in the future, we will have to remove the volatile
// marker.
auto *Result = CGF.Builder.CreateAtomicCmpXchg(
auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
Result->setVolatile(true);
return CGF.Builder.CreateExtractValue(Result, 0);
CmpXchg->setVolatile(true);
auto *Result = CGF.Builder.CreateExtractValue(CmpXchg, 0);
if (RTy->isPointerTy()) {
Result = CGF.Builder.CreateIntToPtr(Result, RTy);
}
return Result;
}
// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
@ -1620,6 +1633,7 @@ enum class CodeGenFunction::MSVCIntrin {
_BitScanForward,
_BitScanReverse,
_InterlockedAnd,
_InterlockedCompareExchange,
_InterlockedDecrement,
_InterlockedExchange,
_InterlockedExchangeAdd,
@ -1705,26 +1719,31 @@ translateArmToMsvcIntrin(unsigned BuiltinID) {
case clang::ARM::BI_InterlockedExchange16_acq:
case clang::ARM::BI_InterlockedExchange_acq:
case clang::ARM::BI_InterlockedExchange64_acq:
case clang::ARM::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::ARM::BI_InterlockedExchange8_rel:
case clang::ARM::BI_InterlockedExchange16_rel:
case clang::ARM::BI_InterlockedExchange_rel:
case clang::ARM::BI_InterlockedExchange64_rel:
case clang::ARM::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::ARM::BI_InterlockedExchange8_nf:
case clang::ARM::BI_InterlockedExchange16_nf:
case clang::ARM::BI_InterlockedExchange_nf:
case clang::ARM::BI_InterlockedExchange64_nf:
case clang::ARM::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::ARM::BI_InterlockedCompareExchange8_acq:
case clang::ARM::BI_InterlockedCompareExchange16_acq:
case clang::ARM::BI_InterlockedCompareExchange_acq:
case clang::ARM::BI_InterlockedCompareExchange64_acq:
case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::ARM::BI_InterlockedCompareExchange8_rel:
case clang::ARM::BI_InterlockedCompareExchange16_rel:
case clang::ARM::BI_InterlockedCompareExchange_rel:
case clang::ARM::BI_InterlockedCompareExchange64_rel:
case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::ARM::BI_InterlockedCompareExchange8_nf:
case clang::ARM::BI_InterlockedCompareExchange16_nf:
@ -1851,26 +1870,31 @@ translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
case clang::AArch64::BI_InterlockedExchange16_acq:
case clang::AArch64::BI_InterlockedExchange_acq:
case clang::AArch64::BI_InterlockedExchange64_acq:
case clang::AArch64::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::AArch64::BI_InterlockedExchange8_rel:
case clang::AArch64::BI_InterlockedExchange16_rel:
case clang::AArch64::BI_InterlockedExchange_rel:
case clang::AArch64::BI_InterlockedExchange64_rel:
case clang::AArch64::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::AArch64::BI_InterlockedExchange8_nf:
case clang::AArch64::BI_InterlockedExchange16_nf:
case clang::AArch64::BI_InterlockedExchange_nf:
case clang::AArch64::BI_InterlockedExchange64_nf:
case clang::AArch64::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::AArch64::BI_InterlockedCompareExchange8_acq:
case clang::AArch64::BI_InterlockedCompareExchange16_acq:
case clang::AArch64::BI_InterlockedCompareExchange_acq:
case clang::AArch64::BI_InterlockedCompareExchange64_acq:
case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::AArch64::BI_InterlockedCompareExchange8_rel:
case clang::AArch64::BI_InterlockedCompareExchange16_rel:
case clang::AArch64::BI_InterlockedCompareExchange_rel:
case clang::AArch64::BI_InterlockedCompareExchange64_rel:
case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::AArch64::BI_InterlockedCompareExchange8_nf:
case clang::AArch64::BI_InterlockedCompareExchange16_nf:
@ -2073,6 +2097,8 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
case MSVCIntrin::_InterlockedExchange_nf:
return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
AtomicOrdering::Monotonic);
case MSVCIntrin::_InterlockedCompareExchange:
return EmitAtomicCmpXchgForMSIntrin(*this, E);
case MSVCIntrin::_InterlockedCompareExchange_acq:
return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
case MSVCIntrin::_InterlockedCompareExchange_rel:
@ -5720,32 +5746,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
case Builtin::BI_InterlockedCompareExchangePointer:
case Builtin::BI_InterlockedCompareExchangePointer_nf: {
llvm::Type *RTy;
llvm::IntegerType *IntType = IntegerType::get(
getLLVMContext(), getContext().getTypeSize(E->getType()));
Address DestAddr = CheckAtomicAlignment(*this, E);
llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
RTy = Exchange->getType();
Exchange = Builder.CreatePtrToInt(Exchange, IntType);
llvm::Value *Comparand =
Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
auto Ordering =
BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
Ordering, Ordering);
Result->setVolatile(true);
return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
0),
RTy));
}
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange, E));
case Builtin::BI_InterlockedCompareExchangePointer_nf:
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E));
case Builtin::BI_InterlockedCompareExchange8:
case Builtin::BI_InterlockedCompareExchange16:
case Builtin::BI_InterlockedCompareExchange:

9
clang/lib/Headers/intrin0.h
View File

@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);
/*----------------------------------------------------------------------------*\
|* Interlocked Compare Exchange
@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
__int64 _Exchange,
__int64 _Comparand);
void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
void *_Exchange, void *_Comparand);
void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
void *_Exchange, void *_Comparand);
void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
void *_Exchange, void *_Comparand);
#endif
#ifdef __cplusplus

66
clang/test/CodeGen/ms-intrinsics.c
View File

@ -221,6 +221,41 @@ void *test_InterlockedExchangePointer(void * volatile *Target, void *Value) {
// CHECK: ret ptr %[[RESULT]]
// CHECK: }
#if defined(__arm__) || defined(__aarch64__)
void *test_InterlockedExchangePointer_acq(void * volatile *Target, void *Value) {
return _InterlockedExchangePointer_acq(Target, Value);
}
// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_acq(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR:i[0-9]+]]
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] acquire, align {{4|8}}
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
// CHECK-ARM-ARM64: }
void *test_InterlockedExchangePointer_nf(void * volatile *Target, void *Value) {
return _InterlockedExchangePointer_nf(Target, Value);
}
// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_nf(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] monotonic, align {{4|8}}
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
// CHECK-ARM-ARM64: }
void *test_InterlockedExchangePointer_rel(void * volatile *Target, void *Value) {
return _InterlockedExchangePointer_rel(Target, Value);
}
// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_rel(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] release, align {{4|8}}
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
// CHECK-ARM-ARM64: }
#endif
void *test_InterlockedCompareExchangePointer(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer(Destination, Exchange, Comparand);
@ -249,6 +284,37 @@ void *test_InterlockedCompareExchangePointer_nf(void * volatile *Destination,
// CHECK: ret ptr %[[RESULT:[0-9]+]]
// CHECK: }
#if defined(__arm__) || defined(__aarch64__)
void *test_InterlockedCompareExchangePointer_acq(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer_acq(Destination, Exchange, Comparand);
}
// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_acq(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] acquire acquire, align {{4|8}}
// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
// CHECK-ARM-ARM64: }
void *test_InterlockedCompareExchangePointer_rel(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer_rel(Destination, Exchange, Comparand);
}
// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_rel(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] release monotonic, align {{4|8}}
// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
// CHECK-ARM-ARM64: }
#endif
char test_InterlockedExchange8(char volatile *value, char mask) {
return _InterlockedExchange8(value, mask);
}