shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/libcxx/include/__atomic/atomic.h
Nikolas Klauser 34d4f0c136
[libc++][NFC] Use llvm.org/PR to link to bug reports (#156288) 
We've built up quite a few links directly to github within the code
base. We should instead use `llvm.org/PR<issue-number>` to link to bugs,
since that is resilient to the bug tracker changing in the future. This
is especially relevant for tests linking to bugs, since they will
probably be there for decades to come. A nice side effect is that these
links are significantly shorter than the GH links, making them much less
of an eyesore.

This patch also replaces a few links that linked to the old bugzilla
instance on llvm.org.
2025-09-04 09:20:02 +02:00

831 lines
33 KiB
C++
Raw Blame History

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _LIBCPP___ATOMIC_ATOMIC_H
#define _LIBCPP___ATOMIC_ATOMIC_H
#include <__atomic/atomic_sync.h>
#include <__atomic/check_memory_order.h>
#include <__atomic/is_always_lock_free.h>
#include <__atomic/memory_order.h>
#include <__atomic/support.h>
#include <__config>
#include <__cstddef/ptrdiff_t.h>
#include <__memory/addressof.h>
#include <__type_traits/enable_if.h>
#include <__type_traits/is_floating_point.h>
#include <__type_traits/is_function.h>
#include <__type_traits/is_integral.h>
#include <__type_traits/is_nothrow_constructible.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_trivially_copyable.h>
#include <__type_traits/remove_const.h>
#include <__type_traits/remove_pointer.h>
#include <__type_traits/remove_volatile.h>
#include <__utility/forward.h>
#include <cstring>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif
_LIBCPP_BEGIN_NAMESPACE_STD
template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
struct __atomic_base // false
{
mutable __cxx_atomic_impl<_Tp> __a_;
using value_type = _Tp;
#if _LIBCPP_STD_VER >= 17
static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value;
#endif
_LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT {
return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));
}
_LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT {
return static_cast<__atomic_base const volatile*>(this)->is_lock_free();
}
_LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
_LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
}
_LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
_LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
return std::__cxx_atomic_load(std::addressof(__a_), __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
return std::__cxx_atomic_load(std::addressof(__a_), __m);
}
_LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); }
_LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); }
_LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
}
_LIBCPP_HIDE_FROM_ABI bool
compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
}
_LIBCPP_HIDE_FROM_ABI bool
compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
}
_LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
_LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
}
_LIBCPP_HIDE_FROM_ABI bool
compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
}
_LIBCPP_HIDE_FROM_ABI bool
compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
}
_LIBCPP_HIDE_FROM_ABI bool
compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
}
_LIBCPP_HIDE_FROM_ABI bool
compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
}
#if _LIBCPP_STD_VER >= 20
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const
volatile _NOEXCEPT {
std::__atomic_wait(*this, __v, __m);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
std::__atomic_wait(*this, __v, __m);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
std::__atomic_notify_one(*this);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT {
std::__atomic_notify_all(*this);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
#endif // _LIBCPP_STD_VER >= 20
#if _LIBCPP_STD_VER >= 20
_LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
#else
_LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default;
#endif
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
__atomic_base(const __atomic_base&) = delete;
};
// atomic<Integral>
template <class _Tp>
struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
using __base _LIBCPP_NODEBUG = __atomic_base<_Tp, false>;
using difference_type = typename __base::value_type;
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default;
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); }
_LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); }
_LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); }
_LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); }
_LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
_LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
_LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
_LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; }
};
// Here we need _IsIntegral because the default template argument is not enough
// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from
// __atomic_base<int, false> and the caller of the wait function is
// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
template <class _Tp, bool _IsIntegral>
struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
return __a.load(__order);
}
static _LIBCPP_HIDE_FROM_ABI _Tp
__atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) {
return __this.load(__order);
}
static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>*
__atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) {
return std::addressof(__a.__a_);
}
static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>*
__atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) {
return std::addressof(__this.__a_);
}
};
template <typename _Tp>
struct __check_atomic_mandates {
using type _LIBCPP_NODEBUG = _Tp;
static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
};
template <class _Tp>
struct atomic : public __atomic_base<typename __check_atomic_mandates<_Tp>::type> {
using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;
#if _LIBCPP_STD_VER >= 20
_LIBCPP_HIDE_FROM_ABI atomic() = default;
#else
_LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;
#endif
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
_LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT {
__base::store(__d);
return __d;
}
_LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT {
__base::store(__d);
return __d;
}
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
};
// atomic<T*>
template <class _Tp>
struct atomic<_Tp*> : public __atomic_base<_Tp*> {
using __base _LIBCPP_NODEBUG = __atomic_base<_Tp*>;
using difference_type = ptrdiff_t;
_LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
_LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT {
__base::store(__d);
return __d;
}
_LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT {
__base::store(__d);
return __d;
}
_LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
// __atomic_fetch_add accepts function pointers, guard against them.
static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
// __atomic_fetch_add accepts function pointers, guard against them.
static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
// __atomic_fetch_add accepts function pointers, guard against them.
static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
// __atomic_fetch_add accepts function pointers, guard against them.
static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); }
_LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); }
_LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); }
_LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); }
_LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; }
_LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; }
_LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; }
_LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; }
_LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
_LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; }
_LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
_LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
};
template <class _Tp>
struct __atomic_waitable_traits<atomic<_Tp> > : __atomic_waitable_traits<__atomic_base<_Tp> > {};
#if _LIBCPP_STD_VER >= 20
template <class _Tp>
requires is_floating_point_v<_Tp>
struct atomic<_Tp> : __atomic_base<_Tp> {
private:
_LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() {
// Only x87-fp80 long double has 64-bit mantissa
return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>;
}
_LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() {
# ifndef _LIBCPP_COMPILER_CLANG_BASED
return false;
# else
// The builtin __cxx_atomic_fetch_add errors during compilation for
// long double on platforms with fp80 format.
// For more details, see
// lib/Sema/SemaChecking.cpp function IsAllowedValueType
// LLVM Parser does not allow atomicrmw with x86_fp80 type.
// if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) &&
// &Context.getTargetInfo().getLongDoubleFormat() ==
// &llvm::APFloat::x87DoubleExtended())
// For more info
// https://llvm.org/PR68602
// https://reviews.llvm.org/D53965
return !__is_fp80_long_double();
# endif
}
template <class _This, class _Operation, class _BuiltinOp>
_LIBCPP_HIDE_FROM_ABI static _Tp
__rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
if constexpr (__has_rmw_builtin()) {
return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
} else {
_Tp __old = __self.load(memory_order_relaxed);
_Tp __new = __operation(__old, __operand);
while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
# ifdef _LIBCPP_COMPILER_CLANG_BASED
if constexpr (__is_fp80_long_double()) {
// https://llvm.org/PR47978
// clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
// Note __old = __self.load(memory_order_relaxed) will not work
std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), std::addressof(__old), memory_order_relaxed);
}
# endif
__new = __operation(__old, __operand);
}
return __old;
}
}
template <class _This>
_LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {
auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
};
auto __plus = [](auto __a, auto __b) { return __a + __b; };
return __rmw_op(std::forward<_This>(__self), __operand, __m, __plus, __builtin_op);
}
template <class _This>
_LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {
auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
};
auto __minus = [](auto __a, auto __b) { return __a - __b; };
return __rmw_op(std::forward<_This>(__self), __operand, __m, __minus, __builtin_op);
}
public:
using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;
using value_type = _Tp;
using difference_type = value_type;
_LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default;
_LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {}
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;
_LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept
requires __base::is_always_lock_free
{
__base::store(__d);
return __d;
}
_LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept {
__base::store(__d);
return __d;
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
requires __base::is_always_lock_free
{
return __fetch_add(*this, __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
return __fetch_add(*this, __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
requires __base::is_always_lock_free
{
return __fetch_sub(*this, __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
return __fetch_sub(*this, __op, __m);
}
_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept
requires __base::is_always_lock_free
{
return fetch_add(__op) + __op;
}
_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }
_LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept
requires __base::is_always_lock_free
{
return fetch_sub(__op) - __op;
}
_LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }
};
#endif // _LIBCPP_STD_VER >= 20
// atomic_is_lock_free
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT {
return __o->is_lock_free();
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT {
return __o->is_lock_free();
}
// atomic_init
template <class _Tp>
_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
}
template <class _Tp>
_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
}
// atomic_store
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
__o->store(__d);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
__o->store(__d);
}
// atomic_store_explicit
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void
atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
_LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
__o->store(__d, __m);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void
atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
_LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
__o->store(__d, __m);
}
// atomic_load
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT {
return __o->load();
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT {
return __o->load();
}
// atomic_load_explicit
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
return __o->load(__m);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
return __o->load(__m);
}
// atomic_exchange
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
return __o->exchange(__d);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
return __o->exchange(__d);
}
// atomic_exchange_explicit
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
return __o->exchange(__d, __m);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
return __o->exchange(__d, __m);
}
// atomic_compare_exchange_weak
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(
volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
return __o->compare_exchange_weak(*__e, __d);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(
atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
return __o->compare_exchange_weak(*__e, __d);
}
// atomic_compare_exchange_strong
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(
volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
return __o->compare_exchange_strong(*__e, __d);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(
atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
return __o->compare_exchange_strong(*__e, __d);
}
// atomic_compare_exchange_weak_explicit
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
volatile atomic<_Tp>* __o,
typename atomic<_Tp>::value_type* __e,
typename atomic<_Tp>::value_type __d,
memory_order __s,
memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
return __o->compare_exchange_weak(*__e, __d, __s, __f);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
atomic<_Tp>* __o,
typename atomic<_Tp>::value_type* __e,
typename atomic<_Tp>::value_type __d,
memory_order __s,
memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
return __o->compare_exchange_weak(*__e, __d, __s, __f);
}
// atomic_compare_exchange_strong_explicit
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
volatile atomic<_Tp>* __o,
typename atomic<_Tp>::value_type* __e,
typename atomic<_Tp>::value_type __d,
memory_order __s,
memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
return __o->compare_exchange_strong(*__e, __d, __s, __f);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
atomic<_Tp>* __o,
typename atomic<_Tp>::value_type* __e,
typename atomic<_Tp>::value_type __d,
memory_order __s,
memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
return __o->compare_exchange_strong(*__e, __d, __s, __f);
}
#if _LIBCPP_STD_VER >= 20
// atomic_wait
template <class _Tp>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
return __o->wait(__v);
}
template <class _Tp>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
return __o->wait(__v);
}
// atomic_wait_explicit
template <class _Tp>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
return __o->wait(__v, __m);
}
template <class _Tp>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
return __o->wait(__v, __m);
}
// atomic_notify_one
template <class _Tp>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {
__o->notify_one();
}
template <class _Tp>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {
__o->notify_one();
}
// atomic_notify_all
template <class _Tp>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {
__o->notify_all();
}
template <class _Tp>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {
__o->notify_all();
}
#endif // _LIBCPP_STD_VER >= 20
// atomic_fetch_add
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
return __o->fetch_add(__op);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
return __o->fetch_add(__op);
}
// atomic_fetch_add_explicit
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit(
volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_add(__op, __m);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_add(__op, __m);
}
// atomic_fetch_sub
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
return __o->fetch_sub(__op);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
return __o->fetch_sub(__op);
}
// atomic_fetch_sub_explicit
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit(
volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_sub(__op, __m);
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_sub(__op, __m);
}
// atomic_fetch_and
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
return __o->fetch_and(__op);
}
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
return __o->fetch_and(__op);
}
// atomic_fetch_and_explicit
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit(
volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_and(__op, __m);
}
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_and(__op, __m);
}
// atomic_fetch_or
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
return __o->fetch_or(__op);
}
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
return __o->fetch_or(__op);
}
// atomic_fetch_or_explicit
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_or(__op, __m);
}
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_or(__op, __m);
}
// atomic_fetch_xor
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
return __o->fetch_xor(__op);
}
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
return __o->fetch_xor(__op);
}
// atomic_fetch_xor_explicit
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit(
volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_xor(__op, __m);
}
template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
_LIBCPP_HIDE_FROM_ABI _Tp
atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
return __o->fetch_xor(__op, __m);
}
_LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP___ATOMIC_ATOMIC_H
Reference in New Issue View Git Blame Copy Permalink