[Clang] Rename uinc_wrap and add normal atomic builtin (#177253)

Summary:
The `__scoped_atomic` builtins are supposed to match the standard
GNU-flavored `__atomic` builtins. We added a scoped builtin without a
corresponding standard one before the fork so this should be added in
the release candidate. These were originally added in
https://github.com/llvm/llvm-project/pull/168666

Also, the name `uinc_wrap` does not follow the naming convention. The
GNU atomics use `fetch_xyz` to indicate that the builtin returns the
previous location's value as part of the RMW operation, which these do.
This PR renames it and its uses.

clang/docs/LanguageExtensions.rst

@@ -4859,6 +4859,20 @@ The syntax and semantics are similar to GCC-compatible __atomic_* builtins.
 The builtins work with signed and unsigned integers and require to specify memory ordering.
 The return value is the original value that was stored in memory before comparison.
 
+Clang provides two additional atomic builtins with incrementing behavior. These
+builtins perform an unsigned increment or decrement modulo a  wrap-around value.
+
+* ``__atomic_fetch_uinc``
+* ``__atomic_fetch_udec``
+
+See the LLVM IR `atomicrmw <https://llvm.org/docs/LangRef.html#atomicrmw-instruction>`_
+instruction for the complete semantics of uinc_wrap and udec_wrap.
+
+Atomic memory scopes are designed to assist optimizations for systems with
+several levels of memory hierarchy like GPUs. The following memory scopes are
+currently supported:
+
+
 Example:
 
 .. code-block:: c
@@ -4926,18 +4940,6 @@ memory scope argument. These are designed to be a generic alternative to the
 ``__opencl_atomic_*`` builtin functions for targets that support atomic memory
 scopes.
 
-Clang provides two additional __scoped_atomic builtins:
-
-* ``__scoped_atomic_uinc_wrap``
-* ``__scoped_atomic_udec_wrap``
-
-See LLVM IR `atomicrmw <https://llvm.org/docs/LangRef.html#atomicrmw-instruction>`_
-instruction for the semantics of uinc_wrap and udec_wrap.
-
-Atomic memory scopes are designed to assist optimizations for systems with
-several levels of memory hierarchy like GPUs. The following memory scopes are
-currently supported:
-
 * ``__MEMORY_SCOPE_SYSTEM``
 * ``__MEMORY_SCOPE_DEVICE``
 * ``__MEMORY_SCOPE_WRKGRP``

clang/include/clang/Basic/Builtins.td

@@ -2314,14 +2314,14 @@ def ScopedAtomicMaxFetch : AtomicBuiltin {
   let Prototype = "void(...)";
 }
 
-def ScopedAtomicUIncWrap : AtomicBuiltin {
-  let Spellings = ["__scoped_atomic_uinc_wrap"];
+def ScopedAtomicUInc : AtomicBuiltin {
+  let Spellings = ["__scoped_atomic_fetch_uinc"];
   let Attributes = [CustomTypeChecking];
   let Prototype = "void(...)";
 }
 
-def ScopedAtomicUDecWrap : AtomicBuiltin {
-  let Spellings = ["__scoped_atomic_udec_wrap"];
+def ScopedAtomicUDec : AtomicBuiltin {
+  let Spellings = ["__scoped_atomic_fetch_udec"];
   let Attributes = [CustomTypeChecking];
   let Prototype = "void(...)";
 }
@@ -2418,6 +2418,18 @@ def AtomicFetchMin : AtomicBuiltin {
   let Prototype = "void(...)";
 }
 
+def AtomicFetchUInc : AtomicBuiltin {
+  let Spellings = ["__atomic_fetch_uinc"];
+  let Attributes = [CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
+def AtomicFetchUDec : AtomicBuiltin {
+  let Spellings = ["__atomic_fetch_udec"];
+  let Attributes = [CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
 // HIP atomic builtins.
 def HipAtomicLoad : AtomicBuiltin {
   let Spellings = ["__hip_atomic_load"];

clang/lib/AST/Expr.cpp

@@ -5196,6 +5196,8 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) {
   case AO__atomic_max_fetch:
   case AO__atomic_fetch_min:
   case AO__atomic_fetch_max:
+  case AO__atomic_fetch_uinc:
+  case AO__atomic_fetch_udec:
     return 3;
 
   case AO__scoped_atomic_load:
@@ -5218,8 +5220,8 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) {
   case AO__scoped_atomic_fetch_min:
   case AO__scoped_atomic_fetch_max:
   case AO__scoped_atomic_exchange_n:
-  case AO__scoped_atomic_uinc_wrap:
-  case AO__scoped_atomic_udec_wrap:
+  case AO__scoped_atomic_fetch_uinc:
+  case AO__scoped_atomic_fetch_udec:
   case AO__hip_atomic_exchange:
   case AO__hip_atomic_fetch_add:
   case AO__hip_atomic_fetch_sub:

clang/lib/CIR/CodeGen/CIRGenAtomic.cpp

@@ -638,8 +638,10 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
 
   case AtomicExpr::AO__scoped_atomic_fetch_nand:
 
-  case AtomicExpr::AO__scoped_atomic_uinc_wrap:
-  case AtomicExpr::AO__scoped_atomic_udec_wrap:
+  case AtomicExpr::AO__scoped_atomic_fetch_uinc:
+  case AtomicExpr::AO__scoped_atomic_fetch_udec:
+  case AtomicExpr::AO__atomic_fetch_uinc:
+  case AtomicExpr::AO__atomic_fetch_udec:
     cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI");
     return;
   }

clang/lib/CodeGen/CGAtomic.cpp

@@ -767,10 +767,12 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
     Op = llvm::AtomicRMWInst::Nand;
     break;
 
-  case AtomicExpr::AO__scoped_atomic_uinc_wrap:
+  case AtomicExpr::AO__atomic_fetch_uinc:
+  case AtomicExpr::AO__scoped_atomic_fetch_uinc:
     Op = llvm::AtomicRMWInst::UIncWrap;
     break;
-  case AtomicExpr::AO__scoped_atomic_udec_wrap:
+  case AtomicExpr::AO__atomic_fetch_udec:
+  case AtomicExpr::AO__scoped_atomic_fetch_udec:
     Op = llvm::AtomicRMWInst::UDecWrap;
     break;
 
@@ -1046,6 +1048,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
   case AtomicExpr::AO__atomic_fetch_nand:
   case AtomicExpr::AO__atomic_fetch_or:
   case AtomicExpr::AO__atomic_fetch_xor:
+  case AtomicExpr::AO__atomic_fetch_uinc:
+  case AtomicExpr::AO__atomic_fetch_udec:
   case AtomicExpr::AO__atomic_and_fetch:
   case AtomicExpr::AO__atomic_nand_fetch:
   case AtomicExpr::AO__atomic_or_fetch:
@@ -1078,8 +1082,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
   case AtomicExpr::AO__scoped_atomic_xor_fetch:
   case AtomicExpr::AO__scoped_atomic_store_n:
   case AtomicExpr::AO__scoped_atomic_exchange_n:
-  case AtomicExpr::AO__scoped_atomic_uinc_wrap:
-  case AtomicExpr::AO__scoped_atomic_udec_wrap:
+  case AtomicExpr::AO__scoped_atomic_fetch_uinc:
+  case AtomicExpr::AO__scoped_atomic_fetch_udec:
     Val1 = EmitValToTemp(*this, E->getVal1());
     break;
   }
@@ -1278,10 +1282,12 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
     case AtomicExpr::AO__opencl_atomic_fetch_max:
     case AtomicExpr::AO__scoped_atomic_fetch_max:
     case AtomicExpr::AO__scoped_atomic_max_fetch:
-    case AtomicExpr::AO__scoped_atomic_uinc_wrap:
-    case AtomicExpr::AO__scoped_atomic_udec_wrap:
+    case AtomicExpr::AO__scoped_atomic_fetch_uinc:
+    case AtomicExpr::AO__scoped_atomic_fetch_udec:
     case AtomicExpr::AO__atomic_test_and_set:
     case AtomicExpr::AO__atomic_clear:
+    case AtomicExpr::AO__atomic_fetch_uinc:
+    case AtomicExpr::AO__atomic_fetch_udec:
       llvm_unreachable("Integral atomic operations always become atomicrmw!");
     }
 

clang/lib/Sema/SemaChecking.cpp

@@ -4608,6 +4608,8 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange,
   case AtomicExpr::AO__atomic_or_fetch:
   case AtomicExpr::AO__atomic_xor_fetch:
   case AtomicExpr::AO__atomic_nand_fetch:
+  case AtomicExpr::AO__atomic_fetch_uinc:
+  case AtomicExpr::AO__atomic_fetch_udec:
   case AtomicExpr::AO__scoped_atomic_fetch_and:
   case AtomicExpr::AO__scoped_atomic_fetch_or:
   case AtomicExpr::AO__scoped_atomic_fetch_xor:
@@ -4616,8 +4618,8 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange,
   case AtomicExpr::AO__scoped_atomic_or_fetch:
   case AtomicExpr::AO__scoped_atomic_xor_fetch:
   case AtomicExpr::AO__scoped_atomic_nand_fetch:
-  case AtomicExpr::AO__scoped_atomic_uinc_wrap:
-  case AtomicExpr::AO__scoped_atomic_udec_wrap:
+  case AtomicExpr::AO__scoped_atomic_fetch_uinc:
+  case AtomicExpr::AO__scoped_atomic_fetch_udec:
     Form = Arithmetic;
     break;
 

clang/test/CodeGen/atomic-ops.c

@@ -255,6 +255,18 @@ _Bool fd4(struct S *a, struct S *b, struct S *c) {
   return __atomic_compare_exchange(a, b, c, 1, 5, 5);
 }
 
+int ui1(int* p) {
+  // CHECK-LABEL: @ui1
+  // CHECK: atomicrmw uinc_wrap ptr {{.*}}, i32 {{.*}} release
+  return __atomic_fetch_uinc(p, 42, memory_order_release);
+}
+
+int ud1(int* p) {
+  // CHECK-LABEL: @ud1
+  // CHECK: atomicrmw udec_wrap ptr {{.*}}, i32 {{.*}} release
+  return __atomic_fetch_udec(p, 42, memory_order_release);
+}
+
 int* fp1(_Atomic(int*) *p) {
   // CHECK-LABEL: @fp1
   // CHECK: load atomic i32, ptr {{.*}} seq_cst, align 4

clang/test/CodeGen/scoped-atomic-ops.c

@@ -4640,8 +4640,8 @@ _Bool fi7e(_Bool *c) {
 // SPIRV-NEXT:    ret void
 //
 void fi8a(unsigned int *a, unsigned int *b) {
-  *b = __scoped_atomic_uinc_wrap(b, ~0U, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE);
-  *a = __scoped_atomic_udec_wrap(a, ~0U, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE);
+  *b = __scoped_atomic_fetch_uinc(b, ~0U, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE);
+  *a = __scoped_atomic_fetch_udec(a, ~0U, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE);
 }
 
 //.

clang/test/Sema/atomic-ops.c

@@ -247,6 +247,9 @@ void f(_Atomic(int) *i, const _Atomic(int) *ci,
   __atomic_fetch_max(P, 3, memory_order_seq_cst); // expected-error {{must be a pointer to integer or supported floating point type}}
   __atomic_fetch_max(p, 3);                       // expected-error {{too few arguments to function call, expected 3, have 2}}
 
+  __atomic_fetch_uinc(F, 1, memory_order_seq_cst); // expected-error {{address argument to atomic operation must be a pointer to integer}}
+  __atomic_fetch_udec(F, 1, memory_order_seq_cst); // expected-error {{address argument to atomic operation must be a pointer to integer}}
+
   __c11_atomic_fetch_and(i, 1, memory_order_seq_cst);
   __c11_atomic_fetch_and(p, 1, memory_order_seq_cst); // expected-error {{must be a pointer to atomic integer}}
   __c11_atomic_fetch_and(f, 1, memory_order_seq_cst); // expected-error {{must be a pointer to atomic integer}}
@@ -591,6 +594,20 @@ void memory_checks(_Atomic(int) *Ap, int *p, int val) {
   (void)__atomic_fetch_min(p, val, memory_order_acq_rel);
   (void)__atomic_fetch_min(p, val, memory_order_seq_cst);
 
+  (void)__atomic_fetch_uinc(p, val, memory_order_relaxed);
+  (void)__atomic_fetch_uinc(p, val, memory_order_acquire);
+  (void)__atomic_fetch_uinc(p, val, memory_order_consume);
+  (void)__atomic_fetch_uinc(p, val, memory_order_release);
+  (void)__atomic_fetch_uinc(p, val, memory_order_acq_rel);
+  (void)__atomic_fetch_uinc(p, val, memory_order_seq_cst);
+
+  (void)__atomic_fetch_udec(p, val, memory_order_relaxed);
+  (void)__atomic_fetch_udec(p, val, memory_order_acquire);
+  (void)__atomic_fetch_udec(p, val, memory_order_consume);
+  (void)__atomic_fetch_udec(p, val, memory_order_release);
+  (void)__atomic_fetch_udec(p, val, memory_order_acq_rel);
+  (void)__atomic_fetch_udec(p, val, memory_order_seq_cst);
+
   (void)__atomic_fetch_max(p, val, memory_order_relaxed);
   (void)__atomic_fetch_max(p, val, memory_order_acquire);
   (void)__atomic_fetch_max(p, val, memory_order_consume);

clang/test/Sema/scoped-atomic-ops.c

@@ -40,8 +40,8 @@ void fi3a(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h, unsign
   *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
   *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
   *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
-  *i = __scoped_atomic_uinc_wrap(i, 1u, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
-  *j = __scoped_atomic_udec_wrap(j, 1u, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
+  *i = __scoped_atomic_fetch_uinc(i, 1u, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
+  *j = __scoped_atomic_fetch_udec(j, 1u, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
 }
 
 void fi3b(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h, unsigned *i, unsigned *j) {
@@ -53,8 +53,8 @@ void fi3b(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h, unsign
   *f = __scoped_atomic_fetch_nand(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
   *g = __scoped_atomic_fetch_min(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
   *h = __scoped_atomic_fetch_max(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
-  *i = __scoped_atomic_uinc_wrap(1, 1u, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
-  *g = __scoped_atomic_udec_wrap(1, 1u, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+  *i = __scoped_atomic_fetch_uinc(1, 1u, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+  *g = __scoped_atomic_fetch_udec(1, 1u, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
 }
 
 void fi3c(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h, unsigned *i, unsigned *j) {
@@ -66,8 +66,8 @@ void fi3c(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h, unsign
   *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}}
   *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}}
   *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}}
-  *i = __scoped_atomic_uinc_wrap(i, 1u, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}}
-  *j = __scoped_atomic_udec_wrap(j, 1u, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}}
+  *i = __scoped_atomic_fetch_uinc(i, 1u, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}}
+  *j = __scoped_atomic_fetch_udec(j, 1u, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}}
 }
 
 void fi3d(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h, unsigned *i, unsigned *j) {
@@ -79,8 +79,8 @@ void fi3d(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h, unsign
   *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}}
   *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}}
   *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}}
-  *i = __scoped_atomic_uinc_wrap(i, 1u, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}}
-  *j = __scoped_atomic_udec_wrap(j, 1u, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}}
+  *i = __scoped_atomic_fetch_uinc(i, 1u, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}}
+  *j = __scoped_atomic_fetch_udec(j, 1u, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}}
 }
 
 void fi3e(float *a, float *b, float *c, float *d, float *e, float *f) {
@@ -88,8 +88,8 @@ void fi3e(float *a, float *b, float *c, float *d, float *e, float *f) {
   *b = __scoped_atomic_fetch_or(b, 1, __ATOMIC_RELAXED, 42); // expected-error {{address argument to atomic operation must be a pointer to integer ('float *' invalid)}}
   *c = __scoped_atomic_fetch_xor(c, 1, __ATOMIC_RELAXED, 42); // expected-error {{address argument to atomic operation must be a pointer to integer ('float *' invalid)}}
   *d = __scoped_atomic_fetch_nand(d, 1, __ATOMIC_RELAXED, 42); // expected-error {{address argument to atomic operation must be a pointer to integer ('float *' invalid)}}
-  *f = __scoped_atomic_uinc_wrap(f, 1u, __ATOMIC_RELAXED, 42); // expected-error {{address argument to atomic operation must be a pointer to integer ('float *' invalid)}}
-  *e = __scoped_atomic_udec_wrap(e, 1u, __ATOMIC_RELAXED, 42); // expected-error {{address argument to atomic operation must be a pointer to integer ('float *' invalid)}}
+  *f = __scoped_atomic_fetch_uinc(f, 1u, __ATOMIC_RELAXED, 42); // expected-error {{address argument to atomic operation must be a pointer to integer ('float *' invalid)}}
+  *e = __scoped_atomic_fetch_udec(e, 1u, __ATOMIC_RELAXED, 42); // expected-error {{address argument to atomic operation must be a pointer to integer ('float *' invalid)}}
 }
 
 int fi4a(int *i) {

libclc/clc/lib/generic/atomic/clc_atomic_dec.cl

@@ -9,7 +9,7 @@
 #include <clc/atomic/clc_atomic_dec.h>
 
 #define __CLC_FUNCTION __clc_atomic_dec
-#define __CLC_IMPL_FUNCTION __scoped_atomic_udec_wrap
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_udec
 #define __CLC_INC_DEC
 
 #define __CLC_BODY <clc_atomic_def.inc>

libclc/clc/lib/generic/atomic/clc_atomic_inc.cl

@@ -9,7 +9,7 @@
 #include <clc/atomic/clc_atomic_inc.h>
 
 #define __CLC_FUNCTION __clc_atomic_inc
-#define __CLC_IMPL_FUNCTION __scoped_atomic_uinc_wrap
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_uinc
 #define __CLC_INC_DEC
 
 #define __CLC_BODY <clc_atomic_def.inc>

openmp/device/include/Synchronization.h

@@ -42,7 +42,7 @@ enum MemScopeTy {
 template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>
 V inc(Ty *Address, V Val, atomic::OrderingTy Ordering,
       MemScopeTy MemScope = MemScopeTy::device) {
-  return __scoped_atomic_uinc_wrap(Address, Val, Ordering, MemScope);
+  return __scoped_atomic_fetch_uinc(Address, Val, Ordering, MemScope);
 }
 
 template <typename Ty, typename V = utils::remove_addrspace_t<Ty>>