From 8947ba017fd8968292e7541a1bbfb82863e54041 Mon Sep 17 00:00:00 2001 From: Wenju He Date: Tue, 25 Nov 2025 08:02:14 +0800 Subject: [PATCH] [libclc] Add atomic_init, atomic_flag_clear and atomic_flag_test_and_set (#168329) --- .../clc/atomic/clc_atomic_flag_clear.h | 24 +++++++ .../clc/atomic/clc_atomic_flag_test_and_set.h | 24 +++++++ libclc/clc/lib/generic/SOURCES | 2 + .../generic/atomic/clc_atomic_flag_clear.cl | 25 +++++++ .../atomic/clc_atomic_flag_test_and_set.cl | 25 +++++++ .../clc/opencl/atomic/atomic_flag_clear.h | 46 +++++++++++++ .../opencl/atomic/atomic_flag_test_and_set.h | 50 ++++++++++++++ .../include/clc/opencl/atomic/atomic_init.h | 24 +++++++ .../include/clc/opencl/atomic/atomic_init.inc | 44 +++++++++++++ libclc/opencl/include/clc/opencl/types.h | 48 ++++++++++++++ libclc/opencl/include/clc/opencl/utils.h | 35 ++++++++++ libclc/opencl/lib/generic/SOURCES | 3 + .../lib/generic/atomic/atomic_flag_clear.cl | 61 +++++++++++++++++ .../atomic/atomic_flag_test_and_set.cl | 66 +++++++++++++++++++ .../opencl/lib/generic/atomic/atomic_init.cl | 18 +++++ .../opencl/lib/generic/atomic/atomic_init.inc | 46 +++++++++++++ 16 files changed, 541 insertions(+) create mode 100644 libclc/clc/include/clc/atomic/clc_atomic_flag_clear.h create mode 100644 libclc/clc/include/clc/atomic/clc_atomic_flag_test_and_set.h create mode 100644 libclc/clc/lib/generic/atomic/clc_atomic_flag_clear.cl create mode 100644 libclc/clc/lib/generic/atomic/clc_atomic_flag_test_and_set.cl create mode 100644 libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h create mode 100644 libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h create mode 100644 libclc/opencl/include/clc/opencl/atomic/atomic_init.h create mode 100644 libclc/opencl/include/clc/opencl/atomic/atomic_init.inc create mode 100644 libclc/opencl/include/clc/opencl/types.h create mode 100644 libclc/opencl/include/clc/opencl/utils.h create mode 100644 libclc/opencl/lib/generic/atomic/atomic_flag_clear.cl create mode 100644 libclc/opencl/lib/generic/atomic/atomic_flag_test_and_set.cl create mode 100644 libclc/opencl/lib/generic/atomic/atomic_init.cl create mode 100644 libclc/opencl/lib/generic/atomic/atomic_init.inc diff --git a/libclc/clc/include/clc/atomic/clc_atomic_flag_clear.h b/libclc/clc/include/clc/atomic/clc_atomic_flag_clear.h new file mode 100644 index 000000000000..fee7c0506abc --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_flag_clear.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FLAG_CLEAR_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FLAG_CLEAR_H__ + +#include + +#define __CLC_DECLARE_ATOMIC_FLAG_CLEAR(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void __clc_atomic_flag_clear( \ + ADDRSPACE int *Ptr, int MemoryOrder, int MemoryScope); + +__CLC_DECLARE_ATOMIC_FLAG_CLEAR(global) +__CLC_DECLARE_ATOMIC_FLAG_CLEAR(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECLARE_ATOMIC_FLAG_CLEAR() +#endif + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FLAG_CLEAR_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_flag_test_and_set.h b/libclc/clc/include/clc/atomic/clc_atomic_flag_test_and_set.h new file mode 100644 index 000000000000..afc373204ad7 --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_flag_test_and_set.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FLAG_TEST_AND_SET_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FLAG_TEST_AND_SET_H__ + +#include + +#define __CLC_DECLARE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL bool __clc_atomic_flag_test_and_set( \ + ADDRSPACE int *Ptr, int MemoryOrder, int MemoryScope); + +__CLC_DECLARE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DECLARE_ATOMIC_FLAG_TEST_AND_SET(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECLARE_ATOMIC_FLAG_TEST_AND_SET() +#endif + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FLAG_TEST_AND_SET_H__ diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES index ee4f771799e8..64fc6b4827a1 100644 --- a/libclc/clc/lib/generic/SOURCES +++ b/libclc/clc/lib/generic/SOURCES @@ -9,6 +9,8 @@ atomic/clc_atomic_fetch_min.cl atomic/clc_atomic_fetch_or.cl atomic/clc_atomic_fetch_sub.cl atomic/clc_atomic_fetch_xor.cl +atomic/clc_atomic_flag_clear.cl +atomic/clc_atomic_flag_test_and_set.cl atomic/clc_atomic_inc.cl atomic/clc_atomic_load.cl atomic/clc_atomic_store.cl diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_flag_clear.cl b/libclc/clc/lib/generic/atomic/clc_atomic_flag_clear.cl new file mode 100644 index 000000000000..e03e63bd82d5 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_flag_clear.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include + +#define __CLC_ATOMIC_FLAG_FALSE 0 + +#define __CLC_DEFINE_ATOMIC_FLAG_CLEAR(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void __clc_atomic_flag_clear( \ + ADDRSPACE int *Ptr, int MemoryOrder, int MemoryScope) { \ + __clc_atomic_store(Ptr, __CLC_ATOMIC_FLAG_FALSE, MemoryOrder, \ + MemoryScope); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_CLEAR(global) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC_FLAG_CLEAR() +#endif diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_flag_test_and_set.cl b/libclc/clc/lib/generic/atomic/clc_atomic_flag_test_and_set.cl new file mode 100644 index 000000000000..4a033e3532af --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_flag_test_and_set.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include + +#define __CLC_ATOMIC_FLAG_TRUE 1 + +#define __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool __clc_atomic_flag_test_and_set( \ + ADDRSPACE int *Ptr, int MemoryOrder, int MemoryScope) { \ + return (bool)__clc_atomic_exchange(Ptr, __CLC_ATOMIC_FLAG_TRUE, \ + MemoryOrder, MemoryScope); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET() +#endif diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h new file mode 100644 index 000000000000..2fcd3eef43a6 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_clear.h @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_CLEAR_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_CLEAR_H__ + +#include +#include + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) +_CLC_OVERLOAD _CLC_DECL void atomic_flag_clear(volatile __global atomic_flag *); +_CLC_OVERLOAD _CLC_DECL void atomic_flag_clear(volatile __local atomic_flag *); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL void atomic_flag_clear(volatile atomic_flag *); +#endif // defined(__opencl_c_generic_address_space) +#endif + +#if defined(__opencl_c_atomic_scope_device) +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile __global atomic_flag *, memory_order); +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile __local atomic_flag *, memory_order); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL void atomic_flag_clear_explicit(volatile atomic_flag *, + memory_order); +#endif // defined(__opencl_c_generic_address_space) +#endif + +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile __global atomic_flag *, memory_order, + memory_scope); +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile __local atomic_flag *, memory_order, + memory_scope); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL void +atomic_flag_clear_explicit(volatile atomic_flag *, memory_order, memory_scope); +#endif // defined(__opencl_c_generic_address_space) + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_CLEAR_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h new file mode 100644 index 000000000000..6e3a8e403d5d --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_flag_test_and_set.h @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_TEST_AND_SET_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_TEST_AND_SET_H__ + +#include +#include + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set(volatile __global atomic_flag *); +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set(volatile __local atomic_flag *); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL bool atomic_flag_test_and_set(volatile atomic_flag *); +#endif // defined(__opencl_c_generic_address_space) +#endif + +#if defined(__opencl_c_atomic_scope_device) +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile __global atomic_flag *, + memory_order); +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile __local atomic_flag *, memory_order); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile atomic_flag *, memory_order); +#endif // defined(__opencl_c_generic_address_space) +#endif + +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile __global atomic_flag *, memory_order, + memory_scope); +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile __local atomic_flag *, memory_order, + memory_scope); +#if defined(__opencl_c_generic_address_space) +_CLC_OVERLOAD _CLC_DECL bool +atomic_flag_test_and_set_explicit(volatile atomic_flag *, memory_order, + memory_scope); +#endif // defined(__opencl_c_generic_address_space) + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FLAG_TEST_AND_SET_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_init.h b/libclc/opencl/include/clc/opencl/atomic/atomic_init.h new file mode 100644 index 000000000000..6a2b938fdd52 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_init.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_INIT_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_INIT_H__ + +#include + +#define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE) + +#define __CLC_BODY +#include + +#define __CLC_BODY +#include + +#undef __CLC_ATOMIC_GENTYPE + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_INIT_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_init.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_init.inc new file mode 100644 index 000000000000..80135fae4b39 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_init.inc @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifdef __CLC_SCALAR + +#if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) && \ + defined(cl_khr_int64_extended_atomics)) +#define __CLC_HAVE_64_ATOMIC +#endif +#if defined(__CLC_FPSIZE) && \ + (__CLC_FPSIZE < 64 || defined(__CLC_HAVE_64_ATOMIC)) +#define __CLC_HAVE_FP_ATOMIC +#endif +#if defined(__CLC_GENSIZE) && \ + ((__CLC_GENSIZE == 32) || \ + (__CLC_GENSIZE == 64 && defined(__CLC_HAVE_64_ATOMIC))) +#define __CLC_HAVE_INT_ATOMIC +#endif +#if defined(__CLC_HAVE_FP_ATOMIC) || defined(__CLC_HAVE_INT_ATOMIC) + +#define __CLC_DECL_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void atomic_init( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value); + +__CLC_DECL_ATOMIC(global) +__CLC_DECL_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECL_ATOMIC() +#endif + +#undef __CLC_DECL_ATOMIC + +#endif // __CLC_HAVE_FP_ATOMIC || __CLC_HAVE_INT_ATOMIC + +#undef __CLC_HAVE_INT_ATOMIC +#undef __CLC_HAVE_FP_ATOMIC +#undef __CLC_HAVE_64_ATOMIC + +#endif // __CLC_SCALAR diff --git a/libclc/opencl/include/clc/opencl/types.h b/libclc/opencl/include/clc/opencl/types.h new file mode 100644 index 000000000000..b1be88f21bda --- /dev/null +++ b/libclc/opencl/include/clc/opencl/types.h @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_TYPES_H__ +#define __CLC_OPENCL_TYPES_H__ + +// Copied from clang/lib/Headers/opencl-c-base.h + +typedef enum memory_scope { + memory_scope_work_item = __OPENCL_MEMORY_SCOPE_WORK_ITEM, + memory_scope_work_group = __OPENCL_MEMORY_SCOPE_WORK_GROUP, + memory_scope_device = __OPENCL_MEMORY_SCOPE_DEVICE, +#if defined(__opencl_c_atomic_scope_all_devices) + memory_scope_all_svm_devices = __OPENCL_MEMORY_SCOPE_ALL_SVM_DEVICES, +#if (__OPENCL_C_VERSION__ >= CL_VERSION_3_0 || __OPENCL_CPP_VERSION__ >= 202100) + memory_scope_all_devices = memory_scope_all_svm_devices, +#endif // (__OPENCL_C_VERSION__ >= CL_VERSION_3_0 || __OPENCL_CPP_VERSION__ >= + // 202100) +#endif // defined(__opencl_c_atomic_scope_all_devices) +/** + * Subgroups have different requirements on forward progress, so just test + * all the relevant macros. + * CL 3.0 sub-groups "they are not guaranteed to make independent forward + * progress" KHR subgroups "Subgroups within a workgroup are independent, make + * forward progress with respect to each other" + */ +#if defined(cl_intel_subgroups) || defined(cl_khr_subgroups) || \ + defined(__opencl_c_subgroups) + memory_scope_sub_group = __OPENCL_MEMORY_SCOPE_SUB_GROUP +#endif +} memory_scope; + +typedef enum memory_order { + memory_order_relaxed = __ATOMIC_RELAXED, + memory_order_acquire = __ATOMIC_ACQUIRE, + memory_order_release = __ATOMIC_RELEASE, + memory_order_acq_rel = __ATOMIC_ACQ_REL, +#if defined(__opencl_c_atomic_order_seq_cst) + memory_order_seq_cst = __ATOMIC_SEQ_CST +#endif +} memory_order; + +#endif // __CLC_OPENCL_TYPES_H__ diff --git a/libclc/opencl/include/clc/opencl/utils.h b/libclc/opencl/include/clc/opencl/utils.h new file mode 100644 index 000000000000..42b948b8d30d --- /dev/null +++ b/libclc/opencl/include/clc/opencl/utils.h @@ -0,0 +1,35 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_UTILS_H__ +#define __CLC_OPENCL_UTILS_H__ + +#include +#include + +// INTEL_FEATURE_PISA +static _CLC_INLINE int __opencl_get_clang_memory_scope(memory_scope scope) { + switch (scope) { + case __OPENCL_MEMORY_SCOPE_WORK_ITEM: + return __MEMORY_SCOPE_SINGLE; +#if defined(cl_intel_subgroups) || defined(cl_khr_subgroups) || \ + defined(__opencl_c_subgroups) + case __OPENCL_MEMORY_SCOPE_SUB_GROUP: + return __MEMORY_SCOPE_WVFRNT; +#endif + case __OPENCL_MEMORY_SCOPE_WORK_GROUP: + return __MEMORY_SCOPE_WRKGRP; + case __OPENCL_MEMORY_SCOPE_DEVICE: + return __MEMORY_SCOPE_DEVICE; + default: + return __MEMORY_SCOPE_SYSTEM; + } +} +// end INTEL_FEATURE_PISA + +#endif // __CLC_OPENCL_UTILS_H__ diff --git a/libclc/opencl/lib/generic/SOURCES b/libclc/opencl/lib/generic/SOURCES index 61757efbcaad..94a333e765b1 100644 --- a/libclc/opencl/lib/generic/SOURCES +++ b/libclc/opencl/lib/generic/SOURCES @@ -22,7 +22,10 @@ atomic/atomic_fetch_min.cl atomic/atomic_fetch_or.cl atomic/atomic_fetch_sub.cl atomic/atomic_fetch_xor.cl +atomic/atomic_flag_clear.cl +atomic/atomic_flag_test_and_set.cl atomic/atomic_inc.cl +atomic/atomic_init.cl atomic/atomic_load.cl atomic/atomic_max.cl atomic/atomic_min.cl diff --git a/libclc/opencl/lib/generic/atomic/atomic_flag_clear.cl b/libclc/opencl/lib/generic/atomic/atomic_flag_clear.cl new file mode 100644 index 000000000000..c9f944903f83 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_flag_clear.cl @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_CLEAR(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void atomic_flag_clear( \ + volatile ADDRSPACE atomic_flag *object) { \ + __clc_atomic_flag_clear((ADDRSPACE int *)object, __ATOMIC_SEQ_CST, \ + __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_CLEAR(global) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR() +#endif + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) + +#if defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void atomic_flag_clear_explicit( \ + volatile ADDRSPACE atomic_flag *object, memory_order order) { \ + __clc_atomic_flag_clear((ADDRSPACE int *)object, order, \ + __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER(global) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER() +#endif + +#endif // defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER_SCOPE(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void atomic_flag_clear_explicit( \ + volatile ADDRSPACE atomic_flag *object, memory_order order, \ + memory_scope scope) { \ + __clc_atomic_flag_clear((ADDRSPACE int *)object, order, \ + __opencl_get_clang_memory_scope(scope)); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER_SCOPE(global) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER_SCOPE(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_CLEAR_ORDER_SCOPE() +#endif diff --git a/libclc/opencl/lib/generic/atomic/atomic_flag_test_and_set.cl b/libclc/opencl/lib/generic/atomic/atomic_flag_test_and_set.cl new file mode 100644 index 000000000000..e58079ee226a --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_flag_test_and_set.cl @@ -0,0 +1,66 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include +#include + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool atomic_flag_test_and_set( \ + volatile ADDRSPACE atomic_flag *object) { \ + return __clc_atomic_flag_test_and_set( \ + (ADDRSPACE int *)object, __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET() +#endif + +#undef __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) + +#if defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool atomic_flag_test_and_set_explicit( \ + volatile ADDRSPACE atomic_flag *object, memory_order order) { \ + return __clc_atomic_flag_test_and_set((ADDRSPACE int *)object, order, \ + __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET() +#endif + +#undef __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET + +#endif // defined(__opencl_c_atomic_scope_device) + +#define __CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF bool atomic_flag_test_and_set_explicit( \ + volatile ADDRSPACE atomic_flag *object, memory_order order, \ + memory_scope scope) { \ + return __clc_atomic_flag_test_and_set( \ + (ADDRSPACE int *)object, order, \ + __opencl_get_clang_memory_scope(scope)); \ + } + +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(global) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET(local) +#if defined(__opencl_c_generic_address_space) +__CLC_DEFINE_ATOMIC_FLAG_TEST_AND_SET() +#endif diff --git a/libclc/opencl/lib/generic/atomic/atomic_init.cl b/libclc/opencl/lib/generic/atomic/atomic_init.cl new file mode 100644 index 000000000000..b688d9b04da7 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_init.cl @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include + +#define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE) + +#define __CLC_BODY +#include + +#define __CLC_BODY +#include diff --git a/libclc/opencl/lib/generic/atomic/atomic_init.inc b/libclc/opencl/lib/generic/atomic/atomic_init.inc new file mode 100644 index 000000000000..2e23df76a8c1 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_init.inc @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifdef __CLC_SCALAR + +#if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) && \ + defined(cl_khr_int64_extended_atomics)) +#define __CLC_HAVE_64_ATOMIC +#endif +#if defined(__CLC_FPSIZE) && \ + (__CLC_FPSIZE < 64 || defined(__CLC_HAVE_64_ATOMIC)) +#define __CLC_HAVE_FP_ATOMIC +#endif +#if defined(__CLC_GENSIZE) && \ + ((__CLC_GENSIZE == 32) || \ + (__CLC_GENSIZE == 64 && defined(__CLC_HAVE_64_ATOMIC))) +#define __CLC_HAVE_INT_ATOMIC +#endif +#if defined(__CLC_HAVE_FP_ATOMIC) || defined(__CLC_HAVE_INT_ATOMIC) + +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void atomic_init( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) { \ + *(ADDRSPACE __CLC_GENTYPE *)Ptr = Value; \ + } + +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC() +#endif + +#undef __CLC_DEFINE_ATOMIC + +#endif // __CLC_HAVE_FP_ATOMIC || __CLC_HAVE_INT_ATOMIC + +#undef __CLC_HAVE_INT_ATOMIC +#undef __CLC_HAVE_FP_ATOMIC +#undef __CLC_HAVE_64_ATOMIC + +#endif // __CLC_SCALAR