[libclc] Add generic implementation of some atomic functions in OpenCL spec section 6.15.12.7 (#146814)

Add corresponding clc functions, which are implemented with clang
__scoped_atomic builtins. OpenCL functions are implemented as a wrapper
over clc functions.

Also change legacy atomic_inc and atomic_dec to re-use the newly added
clc_atomic_inc/dec implementations. llvm-diff only no change to
atomic_inc and atomic_dec in bitcode.

Notes:
* Generic OpenCL built-ins functions uses __ATOMIC_SEQ_CST and
__MEMORY_SCOPE_DEVICE for memory order and memory scope parameters.
* OpenCL atomic_*_explicit, atomic_flag* built-ins are not implemented
yet.
* OpenCL built-ins of atomic_intptr_t, atomic_uintptr_t, atomic_size_t
and atomic_ptrdiff_t types are not implemented yet.
* llvm-diff shows no change to nvptx64--nvidiacl.bc and
amdgcn--amdhsa.bc since __opencl_c_atomic_order_seq_cst and
__opencl_c_atomic_scope_device are not defined in these two targets.

libclc/clc/include/clc/atomic/atomic_decl.inc

@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// MemoryOrder is memory order supported by Clang __scoped_atomic* builtins.
+// MemoryScope is memory scope supported by Clang __scoped_atomic* builtins.
+
+#ifdef __CLC_SCALAR
+#if defined(__CLC_FPSIZE) || (__CLC_GENSIZE >= 32)
+
+#ifdef __CLC_NO_VALUE_ARG
+#define __CLC_DECLARE_ATOMIC(ADDRSPACE)                                        \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder,                  \
+      int MemoryScope);
+#elif defined(__CLC_RETURN_VOID)
+#define __CLC_DECLARE_ATOMIC(ADDRSPACE)                                        \
+  _CLC_OVERLOAD _CLC_DECL void FUNCTION(volatile ADDRSPACE __CLC_GENTYPE *Ptr, \
+                                        __CLC_GENTYPE Value, int MemoryOrder,  \
+                                        int MemoryScope);
+#elif defined(__CLC_COMPARE_EXCHANGE)
+#define __CLC_DECLARE_ATOMIC(ADDRSPACE)                                        \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator,         \
+      __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal,       \
+      int MemoryScope);
+#else
+#define __CLC_DECLARE_ATOMIC(ADDRSPACE)                                        \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value,              \
+      int MemoryOrder, int MemoryScope);
+#endif
+
+__CLC_DECLARE_ATOMIC(global)
+__CLC_DECLARE_ATOMIC(local)
+#if _CLC_GENERIC_AS_SUPPORTED
+__CLC_DECLARE_ATOMIC()
+#endif
+
+#undef __CLC_DECLARE_ATOMIC
+
+#endif // defined(__CLC_FPSIZE) || (__CLC_GENSIZE >= 32)
+#endif // __CLC_SCALAR

libclc/clc/include/clc/atomic/clc_atomic_compare_exchange.h

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_COMPARE_EXCHANGE_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_COMPARE_EXCHANGE_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_compare_exchange
+#define __CLC_COMPARE_EXCHANGE
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_COMPARE_EXCHANGE
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_COMPARE_EXCHANGE_H__

libclc/clc/include/clc/atomic/clc_atomic_dec.h

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_DEC_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_DEC_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_dec
+#define __CLC_NO_VALUE_ARG
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_NO_VALUE_ARG
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_DEC_H__

libclc/clc/include/clc/atomic/clc_atomic_exchange.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_EXCHANGE_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_EXCHANGE_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_exchange
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_EXCHANGE_H__

libclc/clc/include/clc/atomic/clc_atomic_fetch_add.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_ADD_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_ADD_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_fetch_add
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_ADD_H__

libclc/clc/include/clc/atomic/clc_atomic_fetch_and.h

@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_AND_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_AND_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_fetch_and
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_AND_H__

libclc/clc/include/clc/atomic/clc_atomic_fetch_max.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_MAX_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_MAX_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_fetch_max
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_MAX_H__

libclc/clc/include/clc/atomic/clc_atomic_fetch_min.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_MIN_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_MIN_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_fetch_min
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_MIN_H__

libclc/clc/include/clc/atomic/clc_atomic_fetch_or.h

@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_OR_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_OR_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_fetch_or
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_OR_H__

libclc/clc/include/clc/atomic/clc_atomic_fetch_sub.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_SUB_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_SUB_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_fetch_sub
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_SUB_H__

libclc/clc/include/clc/atomic/clc_atomic_fetch_xor.h

@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_XOR_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_XOR_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_fetch_xor
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_XOR_H__

libclc/clc/include/clc/atomic/clc_atomic_inc.h

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_INC_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_INC_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_inc
+#define __CLC_NO_VALUE_ARG
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_NO_VALUE_ARG
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_INC_H__

libclc/clc/include/clc/atomic/clc_atomic_load.h

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_LOAD_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_LOAD_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_load
+#define __CLC_NO_VALUE_ARG
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_NO_VALUE_ARG
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_LOAD_H__

libclc/clc/include/clc/atomic/clc_atomic_store.h

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_ATOMIC_CLC_ATOMIC_STORE_H__
+#define __CLC_ATOMIC_CLC_ATOMIC_STORE_H__
+
+#include <clc/internal/clc.h>
+
+#define FUNCTION __clc_atomic_store
+#define __CLC_RETURN_VOID
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_RETURN_VOID
+#undef FUNCTION
+
+#endif // __CLC_ATOMIC_CLC_ATOMIC_STORE_H__

libclc/clc/lib/generic/SOURCES

@@ -1,4 +1,17 @@
 async/clc_prefetch.cl
+atomic/clc_atomic_compare_exchange.cl
+atomic/clc_atomic_dec.cl
+atomic/clc_atomic_exchange.cl
+atomic/clc_atomic_fetch_add.cl
+atomic/clc_atomic_fetch_and.cl
+atomic/clc_atomic_fetch_max.cl
+atomic/clc_atomic_fetch_min.cl
+atomic/clc_atomic_fetch_or.cl
+atomic/clc_atomic_fetch_sub.cl
+atomic/clc_atomic_fetch_xor.cl
+atomic/clc_atomic_inc.cl
+atomic/clc_atomic_load.cl
+atomic/clc_atomic_store.cl
 common/clc_degrees.cl
 common/clc_radians.cl
 common/clc_sign.cl

libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.cl

@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_compare_exchange.h>
+
+#define __CLC_BODY <clc_atomic_compare_exchange.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc_atomic_compare_exchange.inc>
+#include <clc/math/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc

@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __CLC_SCALAR
+
+#if defined(__SPIR32__) || defined(CLC_NVPTX)
+#if (defined(__CLC_FPSIZE) && __CLC_FPSIZE <= 32) ||                           \
+    (defined(__CLC_GENSIZE) && (__CLC_GENSIZE == 32))
+#define __CLC_HAS_ATOMIC
+#endif
+#else // defined(__SPIR32__) || defined(CLC_NVPTX)
+#if defined(__CLC_FPSIZE) || (__CLC_GENSIZE >= 32)
+#define __CLC_HAS_ATOMIC
+#endif
+#endif // defined(__SPIR32__) || defined(CLC_NVPTX)
+
+#ifdef __CLC_HAS_ATOMIC
+
+#ifdef __CLC_FPSIZE
+
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_atomic_compare_exchange(         \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator,         \
+      __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal,       \
+      int MemoryScope) {                                                       \
+    __CLC_U_GENTYPE Comp = __CLC_AS_U_GENTYPE(Comparator);                     \
+    __scoped_atomic_compare_exchange_n(                                        \
+        (ADDRSPACE __CLC_U_GENTYPE *)Ptr, &Comp, __CLC_AS_U_GENTYPE(Value),    \
+        false, MemoryOrderEqual, MemoryOrderUnequal, MemoryScope);             \
+    return __CLC_AS_GENTYPE(Comp);                                             \
+  }
+
+#else
+
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_atomic_compare_exchange(         \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator,         \
+      __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal,       \
+      int MemoryScope) {                                                       \
+    __scoped_atomic_compare_exchange_n(Ptr, &Comparator, Value, false,         \
+                                       MemoryOrderEqual, MemoryOrderUnequal,   \
+                                       MemoryScope);                           \
+    return Comparator;                                                         \
+  }
+
+#endif // __CLC_FPSIZE
+
+__CLC_DEFINE_ATOMIC(global)
+__CLC_DEFINE_ATOMIC(local)
+#if _CLC_GENERIC_AS_SUPPORTED
+__CLC_DEFINE_ATOMIC()
+#endif
+
+#undef __CLC_DEFINE_ATOMIC
+
+#endif // __CLC_HAS_ATOMIC
+#undef __CLC_HAS_ATOMIC
+
+#endif // __CLC_SCALAR

libclc/clc/lib/generic/atomic/clc_atomic_dec.cl

@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_dec.h>
+
+#define FUNCTION __clc_atomic_dec
+#define __IMPL_FUNCTION __scoped_atomic_fetch_add
+#define __CLC_INC_DEC
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_def.inc

@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __CLC_SCALAR
+
+#if defined(__SPIR32__) || defined(CLC_NVPTX)
+#if (defined(__CLC_FPSIZE) && __CLC_FPSIZE <= 32) ||                           \
+    (defined(__CLC_GENSIZE) && (__CLC_GENSIZE == 32))
+#define __CLC_HAS_ATOMIC
+#endif
+#else // defined(__SPIR32__) || defined(CLC_NVPTX)
+#if defined(__CLC_FPSIZE) || (__CLC_GENSIZE >= 32)
+#define __CLC_HAS_ATOMIC
+#endif
+#endif // defined(__SPIR32__) || defined(CLC_NVPTX)
+
+#ifdef __CLC_HAS_ATOMIC
+
+#ifndef __CLC_PTR_CASTTYPE
+#define __CLC_PTR_CASTTYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_AS_RETTYPE
+#define __CLC_AS_RETTYPE(x) x
+#endif
+
+#ifdef __CLC_NO_VALUE_ARG
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder,                  \
+      int MemoryScope) {                                                       \
+    return __CLC_AS_RETTYPE(__IMPL_FUNCTION(                                   \
+        (ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, MemoryOrder, MemoryScope));       \
+  }
+#elif defined(__CLC_INC_DEC)
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder,                  \
+      int MemoryScope) {                                                       \
+    return __CLC_AS_RETTYPE(                                                   \
+        __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, (__CLC_GENTYPE)1, \
+                        MemoryOrder, MemoryScope));                            \
+  }
+#elif defined(__CLC_RETURN_VOID)
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL void FUNCTION(volatile ADDRSPACE __CLC_GENTYPE *Ptr, \
+                                        __CLC_GENTYPE Value, int MemoryOrder,  \
+                                        int MemoryScope) {                     \
+    __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, Value, MemoryOrder,   \
+                    MemoryScope);                                              \
+  }
+#else
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value,              \
+      int MemoryOrder, int MemoryScope) {                                      \
+    return __CLC_AS_RETTYPE(                                                   \
+        __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, Value,            \
+                        MemoryOrder, MemoryScope));                            \
+  }
+#endif
+
+__CLC_DEFINE_ATOMIC(global)
+__CLC_DEFINE_ATOMIC(local)
+#if _CLC_GENERIC_AS_SUPPORTED
+__CLC_DEFINE_ATOMIC()
+#endif
+
+#undef __CLC_DEFINE_ATOMIC
+
+#endif // __CLC_HAS_ATOMIC
+#undef __CLC_HAS_ATOMIC
+
+#endif // __CLC_SCALAR

libclc/clc/lib/generic/atomic/clc_atomic_exchange.cl

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_exchange.h>
+
+#define FUNCTION __clc_atomic_exchange
+#define __IMPL_FUNCTION __scoped_atomic_exchange_n
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_PTR_CASTTYPE
+#undef __CLC_AS_RETTYPE
+#define __CLC_PTR_CASTTYPE __CLC_BIT_INTN
+#define __CLC_AS_RETTYPE(x) __CLC_AS_GENTYPE(x)
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/math/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_fetch_add.cl

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_fetch_add.h>
+
+#define FUNCTION __clc_atomic_fetch_add
+#define __IMPL_FUNCTION __scoped_atomic_fetch_add
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/math/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_fetch_and.cl

@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_fetch_and.h>
+
+#define FUNCTION __clc_atomic_fetch_and
+#define __IMPL_FUNCTION __scoped_atomic_fetch_and
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_fetch_max.cl

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_fetch_max.h>
+
+#define FUNCTION __clc_atomic_fetch_max
+#define __IMPL_FUNCTION __scoped_atomic_fetch_max
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/math/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_fetch_min.cl

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_fetch_min.h>
+
+#define FUNCTION __clc_atomic_fetch_min
+#define __IMPL_FUNCTION __scoped_atomic_fetch_min
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/math/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_fetch_or.cl

@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_fetch_or.h>
+
+#define FUNCTION __clc_atomic_fetch_or
+#define __IMPL_FUNCTION __scoped_atomic_fetch_or
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_fetch_sub.cl

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_fetch_sub.h>
+
+#define FUNCTION __clc_atomic_fetch_sub
+#define __IMPL_FUNCTION __scoped_atomic_fetch_sub
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/math/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_fetch_xor.cl

@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_fetch_xor.h>
+
+#define FUNCTION __clc_atomic_fetch_xor
+#define __IMPL_FUNCTION __scoped_atomic_fetch_xor
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_inc.cl

@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_inc.h>
+
+#define FUNCTION __clc_atomic_inc
+#define __IMPL_FUNCTION __scoped_atomic_fetch_sub
+#define __CLC_INC_DEC
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_load.cl

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_load.h>
+
+#define FUNCTION __clc_atomic_load
+#define __IMPL_FUNCTION __scoped_atomic_load_n
+#define __CLC_NO_VALUE_ARG
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_PTR_CASTTYPE
+#undef __CLC_AS_RETTYPE
+#define __CLC_PTR_CASTTYPE __CLC_BIT_INTN
+#define __CLC_AS_RETTYPE(x) __CLC_AS_GENTYPE(x)
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/math/gentype.inc>

libclc/clc/lib/generic/atomic/clc_atomic_store.cl

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/atomic/clc_atomic_store.h>
+
+#define FUNCTION __clc_atomic_store
+#define __IMPL_FUNCTION __scoped_atomic_store_n
+#define __CLC_RETURN_VOID
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_PTR_CASTTYPE
+#define __CLC_PTR_CASTTYPE __CLC_BIT_INTN
+
+#define __CLC_BODY <clc_atomic_def.inc>
+#include <clc/math/gentype.inc>

libclc/opencl/include/clc/opencl/atomic/atomic_add.h

@@ -12,6 +12,6 @@
 #include <clc/opencl/opencl-base.h>
 
 #define FUNCTION atomic_add
-#include <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_ADD_H__

libclc/opencl/include/clc/opencl/atomic/atomic_and.h

@@ -12,6 +12,6 @@
 #include <clc/opencl/opencl-base.h>
 
 #define FUNCTION atomic_and
-#include <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_AND_H__

libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__
+
+#define FUNCTION atomic_compare_exchange_strong
+#define __CLC_COMPARE_EXCHANGE
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_COMPARE_EXCHANGE
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__

libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__
+
+#define FUNCTION atomic_compare_exchange_weak
+#define __CLC_COMPARE_EXCHANGE
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_COMPARE_EXCHANGE
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__

libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc

@@ -6,17 +6,55 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_DECLARE_ATOMIC(ADDRSPACE, TYPE)                                  \
-  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+#ifdef __CLC_SCALAR
 
-#define __CLC_DECLARE_ATOMIC_ADDRSPACE(TYPE)                                   \
-  __CLC_DECLARE_ATOMIC(global, TYPE)                                           \
-  __CLC_DECLARE_ATOMIC(local, TYPE)
+#if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) &&         \
+                                 defined(cl_khr_int64_extended_atomics))
+#define HAVE_64_ATOMIC
+#endif
+#if defined(__CLC_FPSIZE) && (__CLC_FPSIZE < 64 || defined(HAVE_64_ATOMIC))
+#define HAVE_FP_ATOMIC
+#endif
+#if defined(__CLC_GENSIZE) &&                                                  \
+    ((__CLC_GENSIZE == 32) ||                                                  \
+     (__CLC_GENSIZE == 64 && defined(HAVE_64_ATOMIC)))
+#define HAVE_INT_ATOMIC
+#endif
+#if defined(HAVE_FP_ATOMIC) || defined(HAVE_INT_ATOMIC)
 
-__CLC_DECLARE_ATOMIC_ADDRSPACE(int)
-__CLC_DECLARE_ATOMIC_ADDRSPACE(uint)
+#define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE)
 
-#undef __CLC_DECLARE_ATOMIC_ADDRSPACE
-#undef __CLC_DECLARE_ATOMIC
+#ifdef __CLC_NO_VALUE_ARG
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr);
+#elif defined(__CLC_RETURN_VOID)
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL void FUNCTION(                                       \
+      volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value);
+#elif defined(__CLC_COMPARE_EXCHANGE)
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr,                            \
+      ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired);
+#else
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+      volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value);
+#endif
 
-#undef FUNCTION
+__CLC_DEFINE_ATOMIC(global)
+__CLC_DEFINE_ATOMIC(local)
+#if _CLC_GENERIC_AS_SUPPORTED
+__CLC_DEFINE_ATOMIC()
+#endif
+
+#undef __CLC_DEFINE_ATOMIC
+
+#endif // HAVE_FP_ATOMIC || HAVE_INT_ATOMIC
+
+#undef HAVE_INT_ATOMIC
+#undef HAVE_FP_ATOMIC
+#undef HAVE_64_ATOMIC
+
+#endif // __CLC_SCALAR

libclc/opencl/include/clc/opencl/atomic/atomic_decl_legacy.inc

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#define __CLC_DECLARE_ATOMIC(ADDRSPACE, TYPE)                                  \
+  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+
+#define __CLC_DECLARE_ATOMIC_ADDRSPACE(TYPE)                                   \
+  __CLC_DECLARE_ATOMIC(global, TYPE)                                           \
+  __CLC_DECLARE_ATOMIC(local, TYPE)
+
+__CLC_DECLARE_ATOMIC_ADDRSPACE(int)
+__CLC_DECLARE_ATOMIC_ADDRSPACE(uint)
+
+#undef __CLC_DECLARE_ATOMIC_ADDRSPACE
+#undef __CLC_DECLARE_ATOMIC
+
+#undef FUNCTION

libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__
+
+#define FUNCTION atomic_exchange
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__

libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__
+
+#define FUNCTION atomic_fetch_add
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__

libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h

@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__
+
+#define FUNCTION atomic_fetch_and
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__

libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__
+
+#define FUNCTION atomic_fetch_max
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__

libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__
+
+#define FUNCTION atomic_fetch_min
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__

libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h

@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__
+
+#define FUNCTION atomic_fetch_or
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__

libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__
+
+#define FUNCTION atomic_fetch_sub
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__

libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h

@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__
+
+#define FUNCTION atomic_fetch_xor
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__

libclc/opencl/include/clc/opencl/atomic/atomic_load.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__
+
+#define FUNCTION atomic_load
+#define __CLC_NO_VALUE_ARG
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_NO_VALUE_ARG
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__

libclc/opencl/include/clc/opencl/atomic/atomic_max.h

@@ -12,6 +12,6 @@
 #include <clc/opencl/opencl-base.h>
 
 #define FUNCTION atomic_max
-#include <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_MAX_H__

libclc/opencl/include/clc/opencl/atomic/atomic_min.h

@@ -12,6 +12,6 @@
 #include <clc/opencl/opencl-base.h>
 
 #define FUNCTION atomic_min
-#include <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_MIN_H__

libclc/opencl/include/clc/opencl/atomic/atomic_or.h

@@ -12,6 +12,6 @@
 #include <clc/opencl/opencl-base.h>
 
 #define FUNCTION atomic_or
-#include <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_OR_H__

libclc/opencl/include/clc/opencl/atomic/atomic_store.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__
+#define __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__
+
+#define FUNCTION atomic_store
+#define __CLC_RETURN_VOID
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/math/gentype.inc>
+
+#undef __CLC_RETURN_VOID
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__

libclc/opencl/include/clc/opencl/atomic/atomic_sub.h

@@ -12,6 +12,6 @@
 #include <clc/opencl/opencl-base.h>
 
 #define FUNCTION atomic_sub
-#include <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_SUB_H__

libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h

@@ -15,6 +15,6 @@
 
 _CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile local float *, float);
 _CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile global float *, float);
-#include <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_XCHG_H__

libclc/opencl/include/clc/opencl/atomic/atomic_xor.h

@@ -12,6 +12,6 @@
 #include <clc/opencl/opencl-base.h>
 
 #define FUNCTION atomic_xor
-#include <clc/opencl/atomic/atomic_decl.inc>
+#include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_XOR_H__

libclc/opencl/lib/generic/SOURCES

@@ -8,6 +8,29 @@ atomic/atom_add.cl
 atomic/atom_and.cl
 atomic/atom_cmpxchg.cl
 atomic/atom_dec.cl
+atomic/atomic_add.cl
+atomic/atomic_and.cl
+atomic/atomic_cmpxchg.cl
+atomic/atomic_compare_exchange_strong.cl
+atomic/atomic_compare_exchange_weak.cl
+atomic/atomic_dec.cl
+atomic/atomic_exchange.cl
+atomic/atomic_fetch_add.cl
+atomic/atomic_fetch_and.cl
+atomic/atomic_fetch_max.cl
+atomic/atomic_fetch_min.cl
+atomic/atomic_fetch_or.cl
+atomic/atomic_fetch_sub.cl
+atomic/atomic_fetch_xor.cl
+atomic/atomic_inc.cl
+atomic/atomic_load.cl
+atomic/atomic_max.cl
+atomic/atomic_min.cl
+atomic/atomic_or.cl
+atomic/atomic_store.cl
+atomic/atomic_sub.cl
+atomic/atomic_xchg.cl
+atomic/atomic_xor.cl
 atomic/atom_inc.cl
 atomic/atom_max.cl
 atomic/atom_min.cl
@@ -15,17 +38,6 @@ atomic/atom_or.cl
 atomic/atom_sub.cl
 atomic/atom_xchg.cl
 atomic/atom_xor.cl
-atomic/atomic_add.cl
-atomic/atomic_and.cl
-atomic/atomic_cmpxchg.cl
-atomic/atomic_dec.cl
-atomic/atomic_inc.cl
-atomic/atomic_max.cl
-atomic/atomic_min.cl
-atomic/atomic_or.cl
-atomic/atomic_sub.cl
-atomic/atomic_xchg.cl
-atomic/atomic_xor.cl
 common/degrees.cl
 common/mix.cl
 common/radians.cl

libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_compare_exchange.h>
+#include <clc/opencl/atomic/atomic_compare_exchange_strong.h>
+
+#define FUNCTION atomic_compare_exchange_strong
+#define __CLC_COMPARE_EXCHANGE
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_compare_exchange.h>
+#include <clc/opencl/atomic/atomic_compare_exchange_weak.h>
+
+#define FUNCTION atomic_compare_exchange_weak
+#define __CLC_COMPARE_EXCHANGE
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_dec.cl

@@ -6,15 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <clc/atomic/clc_atomic_dec.h>
 #include <clc/opencl/atomic/atomic_dec.h>
 
-#define IMPL(TYPE, AS)                                                         \
-  _CLC_OVERLOAD _CLC_DEF TYPE atomic_dec(volatile AS TYPE *p) {                \
-    return __sync_fetch_and_sub(p, (TYPE)1);                                   \
-  }
+#define FUNCTION atomic_dec
+#define __IMPL_FUNCTION __clc_atomic_dec
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+#define __CLC_BODY <atomic_inc_dec.inc>
+#include <clc/integer/gentype.inc>

libclc/opencl/lib/generic/atomic/atomic_def.inc

@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __CLC_SCALAR
+
+#if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) &&         \
+                                 defined(cl_khr_int64_extended_atomics))
+#define HAVE_64_ATOMIC
+#endif
+#if defined(__CLC_FPSIZE) && (__CLC_FPSIZE < 64 || defined(HAVE_64_ATOMIC)
+#define HAVE_FP_ATOMIC
+#endif
+#if defined(__CLC_GENSIZE) &&                                                  \
+    ((__CLC_GENSIZE == 32) ||                                                  \
+     (__CLC_GENSIZE == 64 && defined(HAVE_64_ATOMIC)))
+#define HAVE_INT_ATOMIC
+#endif
+#if defined(HAVE_FP_ATOMIC) || defined(HAVE_INT_ATOMIC)
+
+#define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE)
+
+#ifdef __CLC_NO_VALUE_ARG
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(                               \
+      volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr) {                          \
+    return __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr,            \
+                           __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);           \
+  }
+#elif defined(__CLC_RETURN_VOID)
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DEF void FUNCTION(                                        \
+      volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) {     \
+    __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value,            \
+                    __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);                  \
+  }
+#elif defined(__CLC_COMPARE_EXCHANGE)
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(                               \
+      volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr,                            \
+      ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired) {              \
+    __CLC_GENTYPE Comparator = *Expected;                                      \
+    __CLC_GENTYPE RetValue = __clc_atomic_compare_exchange(                    \
+        (volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Comparator, Desired,          \
+        __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE);            \
+    if (Comparator != RetValue) {                                              \
+      *Expected = RetValue;                                                    \
+      return true;                                                             \
+    }                                                                          \
+    return false;                                                              \
+  }
+#else
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(                               \
+      volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) {     \
+    return __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value,     \
+                           __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);           \
+  }
+#endif
+
+__CLC_DEFINE_ATOMIC(global)
+__CLC_DEFINE_ATOMIC(local)
+#if _CLC_GENERIC_AS_SUPPORTED
+__CLC_DEFINE_ATOMIC()
+#endif
+
+#undef __CLC_DEFINE_ATOMIC
+
+#endif // HAVE_FP_ATOMIC || HAVE_INT_ATOMIC
+
+#undef HAVE_INT_ATOMIC
+#undef HAVE_FP_ATOMIC
+#undef HAVE_64_ATOMIC
+
+#endif // __CLC_SCALAR

libclc/opencl/lib/generic/atomic/atomic_exchange.cl

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_exchange.h>
+#include <clc/opencl/atomic/atomic_exchange.h>
+
+#define FUNCTION atomic_exchange
+#define __IMPL_FUNCTION __clc_atomic_exchange
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_fetch_add.h>
+#include <clc/opencl/atomic/atomic_fetch_add.h>
+
+#define FUNCTION atomic_fetch_add
+#define __IMPL_FUNCTION __clc_atomic_fetch_add
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_fetch_and.h>
+#include <clc/opencl/atomic/atomic_fetch_and.h>
+
+#define FUNCTION atomic_fetch_and
+#define __IMPL_FUNCTION __clc_atomic_fetch_and
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_fetch_max.h>
+#include <clc/opencl/atomic/atomic_fetch_max.h>
+
+#define FUNCTION atomic_fetch_max
+#define __IMPL_FUNCTION __clc_atomic_fetch_max
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_fetch_min.h>
+#include <clc/opencl/atomic/atomic_fetch_min.h>
+
+#define FUNCTION atomic_fetch_min
+#define __IMPL_FUNCTION __clc_atomic_fetch_min
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_fetch_or.h>
+#include <clc/opencl/atomic/atomic_fetch_or.h>
+
+#define FUNCTION atomic_fetch_or
+#define __IMPL_FUNCTION __clc_atomic_fetch_or
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_fetch_sub.h>
+#include <clc/opencl/atomic/atomic_fetch_sub.h>
+
+#define FUNCTION atomic_fetch_sub
+#define __IMPL_FUNCTION __clc_atomic_fetch_sub
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl

@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_fetch_xor.h>
+#include <clc/opencl/atomic/atomic_fetch_xor.h>
+
+#define FUNCTION atomic_fetch_xor
+#define __IMPL_FUNCTION __clc_atomic_fetch_xor
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_inc.cl

@@ -6,15 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <clc/atomic/clc_atomic_inc.h>
 #include <clc/opencl/atomic/atomic_inc.h>
 
-#define IMPL(TYPE, AS)                                                         \
-  _CLC_OVERLOAD _CLC_DEF TYPE atomic_inc(volatile AS TYPE *p) {                \
-    return __sync_fetch_and_add(p, (TYPE)1);                                   \
-  }
+#define FUNCTION atomic_inc
+#define __IMPL_FUNCTION __clc_atomic_inc
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+#define __CLC_BODY <atomic_inc_dec.inc>
+#include <clc/integer/gentype.inc>

libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef __CLC_SCALAR
+
+#if __CLC_GENSIZE == 32
+
+#define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(                               \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr) {                                 \
+    return __IMPL_FUNCTION(Ptr, __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);      \
+  }
+
+__CLC_DEFINE_ATOMIC(global)
+__CLC_DEFINE_ATOMIC(local)
+
+#undef __CLC_DEFINE_ATOMIC
+
+#endif // __CLC_GENSIZE == 32
+
+#endif // __CLC_SCALAR

libclc/opencl/lib/generic/atomic/atomic_load.cl

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_load.h>
+#include <clc/opencl/atomic/atomic_load.h>
+
+#define FUNCTION atomic_load
+#define __IMPL_FUNCTION __clc_atomic_load
+#define __CLC_NO_VALUE_ARG
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)

libclc/opencl/lib/generic/atomic/atomic_store.cl

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__opencl_c_atomic_order_seq_cst) &&                                \
+    defined(__opencl_c_atomic_scope_device)
+
+#include <clc/atomic/clc_atomic_store.h>
+#include <clc/opencl/atomic/atomic_store.h>
+
+#define FUNCTION atomic_store
+#define __IMPL_FUNCTION __clc_atomic_store
+#define __CLC_RETURN_VOID
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/integer/gentype.inc>
+
+#define __CLC_BODY <atomic_def.inc>
+#include <clc/math/gentype.inc>
+
+#endif // defined(__opencl_c_atomic_order_seq_cst) &&
+       // defined(__opencl_c_atomic_scope_device)