shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[CIR] Add atomic load and store operations (#153814)

Browse Source 
This patch adds support for atomic loads and stores. Specifically, it
adds support for the following intrinsic calls:

- `__atomic_load` and `__atomic_store`;
- `__c11_atomic_load` and `__c11_atomic_store`.
This commit is contained in:
Sirui Mu 2025-08-20 21:49:04 +08:00 committed by GitHub
parent 2330fd2f73
commit 318b0dda7c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 522 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
View File

@ -161,9 +161,8 @@ public:
uint64_t alignment = 0) {
mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment);
assert(!cir::MissingFeatures::opLoadStoreVolatile());
assert(!cir::MissingFeatures::opLoadStoreMemOrder());
return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false,
alignmentAttr);
alignmentAttr, cir::MemOrderAttr{});
}
mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr,
@ -245,8 +244,10 @@ public:
}
cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst,
mlir::IntegerAttr align = {}) {
return create<cir::StoreOp>(loc, val, dst, align);
bool isVolatile = false,
mlir::IntegerAttr align = {},
cir::MemOrderAttr order = {}) {
return cir::StoreOp::create(*this, loc, val, dst, align, order);
}
[[nodiscard]] cir::GlobalOp createGlobal(mlir::ModuleOp mlirModule,
@ -269,7 +270,8 @@ public:
clang::CharUnits alignment) {
mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment);
auto addr = createAlloca(loc, getPointerTo(type), type, {}, alignmentAttr);
return create<cir::LoadOp>(loc, addr, /*isDeref=*/false, alignmentAttr);
return cir::LoadOp::create(*this, loc, addr, /*isDeref=*/false,
alignmentAttr, /*mem_order=*/{});
}
cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base,

23
clang/include/clang/CIR/Dialect/IR/CIROps.td
View File

@ -299,6 +299,20 @@ def CIR_ConstantOp : CIR_Op<"const", [
let hasFolder = 1;
}
//===----------------------------------------------------------------------===//
// C/C++ memory order definitions
//===----------------------------------------------------------------------===//
def CIR_MemOrder : CIR_I32EnumAttr<
"MemOrder", "Memory order according to C++11 memory model", [
I32EnumAttrCase<"Relaxed", 0, "relaxed">,
I32EnumAttrCase<"Consume", 1, "consume">,
I32EnumAttrCase<"Acquire", 2, "acquire">,
I32EnumAttrCase<"Release", 3, "release">,
I32EnumAttrCase<"AcquireRelease", 4, "acq_rel">,
I32EnumAttrCase<"SequentiallyConsistent", 5, "seq_cst">
]>;
//===----------------------------------------------------------------------===//
// AllocaOp
//===----------------------------------------------------------------------===//
@ -408,13 +422,14 @@ def CIR_LoadOp : CIR_Op<"load", [
let arguments = (ins Arg<CIR_PointerType, "the address to load from",
[MemRead]>:$addr,
UnitAttr:$isDeref,
OptionalAttr<I64Attr>:$alignment
);
OptionalAttr<I64Attr>:$alignment,
OptionalAttr<CIR_MemOrder>:$mem_order);
let results = (outs CIR_AnyType:$result);
let assemblyFormat = [{
(`deref` $isDeref^)?
(`align` `(` $alignment^ `)`)?
(`atomic` `(` $mem_order^ `)`)?
$addr `:` qualified(type($addr)) `,` type($result) attr-dict
}];
@ -451,10 +466,12 @@ def CIR_StoreOp : CIR_Op<"store", [
let arguments = (ins CIR_AnyType:$value,
Arg<CIR_PointerType, "the address to store the value",
[MemWrite]>:$addr,
OptionalAttr<I64Attr>:$alignment);
OptionalAttr<I64Attr>:$alignment,
OptionalAttr<CIR_MemOrder>:$mem_order);
let assemblyFormat = [{
(`align` `(` $alignment^ `)`)?
(`atomic` `(` $mem_order^ `)`)?
$value `,` $addr attr-dict `:` type($value) `,` qualified(type($addr))
}];

12
clang/include/clang/CIR/Dialect/IR/CIROpsEnums.h
View File

@ -113,6 +113,18 @@ LLVM_ATTRIBUTE_UNUSED static bool isValidLinkage(GlobalLinkageKind gl) {
isLinkOnceLinkage(gl);
}
bool operator<(cir::MemOrder, cir::MemOrder) = delete;
bool operator>(cir::MemOrder, cir::MemOrder) = delete;
bool operator<=(cir::MemOrder, cir::MemOrder) = delete;
bool operator>=(cir::MemOrder, cir::MemOrder) = delete;
// Validate an integral value which isn't known to fit within the enum's range
// is a valid AtomicOrderingCABI.
template <typename Int> inline bool isValidCIRAtomicOrderingCABI(Int value) {
return static_cast<Int>(cir::MemOrder::Relaxed) <= value &&
value <= static_cast<Int>(cir::MemOrder::SequentiallyConsistent);
}
} // namespace cir
#endif // CLANG_CIR_DIALECT_IR_CIROPSENUMS_H

3
clang/include/clang/CIR/MissingFeatures.h
View File

@ -49,7 +49,6 @@ struct MissingFeatures {
static bool opLoadEmitScalarRangeCheck() { return false; }
static bool opLoadBooleanRepresentation() { return false; }
static bool opLoadStoreTbaa() { return false; }
static bool opLoadStoreMemOrder() { return false; }
static bool opLoadStoreVolatile() { return false; }
static bool opLoadStoreAtomic() { return false; }
static bool opLoadStoreObjC() { return false; }
@ -163,6 +162,8 @@ struct MissingFeatures {
static bool atomicInfoGetAtomicPointer() { return false; }
static bool atomicInfoGetAtomicAddress() { return false; }
static bool atomicUseLibCall() { return false; }
static bool atomicScope() { return false; }
static bool atomicSyncScopeID() { return false; }
// Misc
static bool abiArgInfo() { return false; }

6
clang/lib/CIR/CodeGen/Address.h
View File

@ -68,6 +68,12 @@ public:
return pointerAndKnownNonNull.getPointer() != nullptr;
}
/// Return address with different pointer, but same element type and
/// alignment.
Address withPointer(mlir::Value newPtr) const {
return Address(newPtr, getElementType(), getAlignment());
}
/// Return address with different element type, a bitcast pointer, and
/// the same alignment.
Address withElementType(CIRGenBuilderTy &builder, mlir::Type ElemTy) const;

332
clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
View File

@ -96,6 +96,15 @@ public:
bool emitMemSetZeroIfNecessary() const;
/// Cast the given pointer to an integer pointer suitable for atomic
/// operations on the source.
Address castToAtomicIntPointer(Address addr) const;
/// If addr is compatible with the iN that will be used for an atomic
/// operation, bitcast it. Otherwise, create a temporary that is suitable and
/// copy the value across.
Address convertToAtomicIntPointer(Address addr) const;
/// Copy an atomic r-value into atomic-layout memory.
void emitCopyIntoMemory(RValue rvalue) const;
@ -111,11 +120,24 @@ public:
return LValue::makeAddr(addr, getValueType(), lvalue.getBaseInfo());
}
/// Creates temp alloca for intermediate operations on atomic value.
Address createTempAlloca() const;
private:
bool requiresMemSetZero(mlir::Type ty) const;
};
} // namespace
// This function emits any expression (scalar, complex, or aggregate)
// into a temporary alloca.
static Address emitValToTemp(CIRGenFunction &cgf, Expr *e) {
Address declPtr = cgf.createMemTemp(
e->getType(), cgf.getLoc(e->getSourceRange()), ".atomictmp");
cgf.emitAnyExprToMem(e, declPtr, e->getType().getQualifiers(),
/*Init*/ true);
return declPtr;
}
/// Does a store of the given IR type modify the full expected width?
static bool isFullSizeType(CIRGenModule &cgm, mlir::Type ty,
uint64_t expectedSize) {
@ -147,6 +169,41 @@ bool AtomicInfo::requiresMemSetZero(mlir::Type ty) const {
llvm_unreachable("bad evaluation kind");
}
Address AtomicInfo::convertToAtomicIntPointer(Address addr) const {
mlir::Type ty = addr.getElementType();
uint64_t sourceSizeInBits = cgf.cgm.getDataLayout().getTypeSizeInBits(ty);
if (sourceSizeInBits != atomicSizeInBits) {
cgf.cgm.errorNYI(
loc,
"AtomicInfo::convertToAtomicIntPointer: convert through temp alloca");
}
return castToAtomicIntPointer(addr);
}
Address AtomicInfo::createTempAlloca() const {
Address tempAlloca = cgf.createMemTemp(
(lvalue.isBitField() && valueSizeInBits > atomicSizeInBits) ? valueTy
: atomicTy,
getAtomicAlignment(), loc, "atomic-temp");
// Cast to pointer to value type for bitfields.
if (lvalue.isBitField()) {
cgf.cgm.errorNYI(loc, "AtomicInfo::createTempAlloca: bitfield lvalue");
}
return tempAlloca;
}
Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
auto intTy = mlir::dyn_cast<cir::IntType>(addr.getElementType());
// Don't bother with int casts if the integer size is the same.
if (intTy && intTy.getWidth() == atomicSizeInBits)
return addr;
auto ty = cgf.getBuilder().getUIntNTy(atomicSizeInBits);
return addr.withElementType(cgf.getBuilder(), ty);
}
bool AtomicInfo::emitMemSetZeroIfNecessary() const {
assert(lvalue.isSimple());
Address addr = lvalue.getAddress();
@ -187,12 +244,185 @@ void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
}
}
static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
Address ptr, Address val1, uint64_t size,
cir::MemOrder order) {
std::unique_ptr<AtomicScopeModel> scopeModel = expr->getScopeModel();
if (scopeModel) {
assert(!cir::MissingFeatures::atomicScope());
cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: atomic scope");
return;
}
assert(!cir::MissingFeatures::atomicSyncScopeID());
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Location loc = cgf.getLoc(expr->getSourceRange());
auto orderAttr = cir::MemOrderAttr::get(builder.getContext(), order);
switch (expr->getOp()) {
case AtomicExpr::AO__c11_atomic_init:
llvm_unreachable("already handled!");
case AtomicExpr::AO__c11_atomic_load:
case AtomicExpr::AO__atomic_load: {
cir::LoadOp load =
builder.createLoad(loc, ptr, /*isVolatile=*/expr->isVolatile());
assert(!cir::MissingFeatures::atomicSyncScopeID());
load->setAttr("mem_order", orderAttr);
builder.createStore(loc, load->getResult(0), dest);
return;
}
case AtomicExpr::AO__c11_atomic_store:
case AtomicExpr::AO__atomic_store: {
cir::LoadOp loadVal1 = builder.createLoad(loc, val1);
assert(!cir::MissingFeatures::atomicSyncScopeID());
builder.createStore(loc, loadVal1, ptr, expr->isVolatile(),
/*align=*/mlir::IntegerAttr{}, orderAttr);
return;
}
case AtomicExpr::AO__opencl_atomic_init:
case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
case AtomicExpr::AO__atomic_compare_exchange:
case AtomicExpr::AO__atomic_compare_exchange_n:
case AtomicExpr::AO__scoped_atomic_compare_exchange:
case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
case AtomicExpr::AO__opencl_atomic_load:
case AtomicExpr::AO__hip_atomic_load:
case AtomicExpr::AO__atomic_load_n:
case AtomicExpr::AO__scoped_atomic_load_n:
case AtomicExpr::AO__scoped_atomic_load:
case AtomicExpr::AO__opencl_atomic_store:
case AtomicExpr::AO__hip_atomic_store:
case AtomicExpr::AO__atomic_store_n:
case AtomicExpr::AO__scoped_atomic_store:
case AtomicExpr::AO__scoped_atomic_store_n:
case AtomicExpr::AO__c11_atomic_exchange:
case AtomicExpr::AO__hip_atomic_exchange:
case AtomicExpr::AO__opencl_atomic_exchange:
case AtomicExpr::AO__atomic_exchange_n:
case AtomicExpr::AO__atomic_exchange:
case AtomicExpr::AO__scoped_atomic_exchange_n:
case AtomicExpr::AO__scoped_atomic_exchange:
case AtomicExpr::AO__atomic_add_fetch:
case AtomicExpr::AO__scoped_atomic_add_fetch:
case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__hip_atomic_fetch_add:
case AtomicExpr::AO__opencl_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
case AtomicExpr::AO__scoped_atomic_fetch_add:
case AtomicExpr::AO__atomic_sub_fetch:
case AtomicExpr::AO__scoped_atomic_sub_fetch:
case AtomicExpr::AO__c11_atomic_fetch_sub:
case AtomicExpr::AO__hip_atomic_fetch_sub:
case AtomicExpr::AO__opencl_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
case AtomicExpr::AO__scoped_atomic_fetch_sub:
case AtomicExpr::AO__atomic_min_fetch:
case AtomicExpr::AO__scoped_atomic_min_fetch:
case AtomicExpr::AO__c11_atomic_fetch_min:
case AtomicExpr::AO__hip_atomic_fetch_min:
case AtomicExpr::AO__opencl_atomic_fetch_min:
case AtomicExpr::AO__atomic_fetch_min:
case AtomicExpr::AO__scoped_atomic_fetch_min:
case AtomicExpr::AO__atomic_max_fetch:
case AtomicExpr::AO__scoped_atomic_max_fetch:
case AtomicExpr::AO__c11_atomic_fetch_max:
case AtomicExpr::AO__hip_atomic_fetch_max:
case AtomicExpr::AO__opencl_atomic_fetch_max:
case AtomicExpr::AO__atomic_fetch_max:
case AtomicExpr::AO__scoped_atomic_fetch_max:
case AtomicExpr::AO__atomic_and_fetch:
case AtomicExpr::AO__scoped_atomic_and_fetch:
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__hip_atomic_fetch_and:
case AtomicExpr::AO__opencl_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
case AtomicExpr::AO__scoped_atomic_fetch_and:
case AtomicExpr::AO__atomic_or_fetch:
case AtomicExpr::AO__scoped_atomic_or_fetch:
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__hip_atomic_fetch_or:
case AtomicExpr::AO__opencl_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
case AtomicExpr::AO__scoped_atomic_fetch_or:
case AtomicExpr::AO__atomic_xor_fetch:
case AtomicExpr::AO__scoped_atomic_xor_fetch:
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__hip_atomic_fetch_xor:
case AtomicExpr::AO__opencl_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
case AtomicExpr::AO__scoped_atomic_fetch_xor:
case AtomicExpr::AO__atomic_nand_fetch:
case AtomicExpr::AO__scoped_atomic_nand_fetch:
case AtomicExpr::AO__c11_atomic_fetch_nand:
case AtomicExpr::AO__atomic_fetch_nand:
case AtomicExpr::AO__scoped_atomic_fetch_nand:
case AtomicExpr::AO__atomic_test_and_set:
case AtomicExpr::AO__atomic_clear:
cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI");
break;
}
}
static bool isMemOrderValid(uint64_t order, bool isStore, bool isLoad) {
if (!cir::isValidCIRAtomicOrderingCABI(order))
return false;
auto memOrder = static_cast<cir::MemOrder>(order);
if (isStore)
return memOrder != cir::MemOrder::Consume &&
memOrder != cir::MemOrder::Acquire &&
memOrder != cir::MemOrder::AcquireRelease;
if (isLoad)
return memOrder != cir::MemOrder::Release &&
memOrder != cir::MemOrder::AcquireRelease;
return true;
}
RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
QualType atomicTy = e->getPtr()->getType()->getPointeeType();
QualType memTy = atomicTy;
if (const auto *ty = atomicTy->getAs<AtomicType>())
memTy = ty->getValueType();
Address val1 = Address::invalid();
Address dest = Address::invalid();
Address ptr = emitPointerWithAlignment(e->getPtr());
assert(!cir::MissingFeatures::openCL());
@ -202,9 +432,105 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
return RValue::get(nullptr);
}
assert(!cir::MissingFeatures::atomicExpr());
cgm.errorNYI(e->getSourceRange(), "atomic expr is NYI");
return RValue::get(nullptr);
TypeInfoChars typeInfo = getContext().getTypeInfoInChars(atomicTy);
uint64_t size = typeInfo.Width.getQuantity();
Expr::EvalResult orderConst;
mlir::Value order;
if (!e->getOrder()->EvaluateAsInt(orderConst, getContext()))
order = emitScalarExpr(e->getOrder());
bool shouldCastToIntPtrTy = true;
switch (e->getOp()) {
default:
cgm.errorNYI(e->getSourceRange(), "atomic op NYI");
return RValue::get(nullptr);
case AtomicExpr::AO__c11_atomic_init:
llvm_unreachable("already handled above with emitAtomicInit");
case AtomicExpr::AO__c11_atomic_load:
break;
case AtomicExpr::AO__atomic_load:
dest = emitPointerWithAlignment(e->getVal1());
break;
case AtomicExpr::AO__atomic_store:
val1 = emitPointerWithAlignment(e->getVal1());
break;
case AtomicExpr::AO__c11_atomic_store:
val1 = emitValToTemp(*this, e->getVal1());
break;
}
QualType resultTy = e->getType().getUnqualifiedType();
// The inlined atomics only function on iN types, where N is a power of 2. We
// need to make sure (via temporaries if necessary) that all incoming values
// are compatible.
LValue atomicValue = makeAddrLValue(ptr, atomicTy);
AtomicInfo atomics(*this, atomicValue, getLoc(e->getSourceRange()));
if (shouldCastToIntPtrTy) {
ptr = atomics.castToAtomicIntPointer(ptr);
if (val1.isValid())
val1 = atomics.convertToAtomicIntPointer(val1);
}
if (dest.isValid()) {
if (shouldCastToIntPtrTy)
dest = atomics.castToAtomicIntPointer(dest);
} else if (!resultTy->isVoidType()) {
dest = atomics.createTempAlloca();
if (shouldCastToIntPtrTy)
dest = atomics.castToAtomicIntPointer(dest);
}
bool powerOf2Size = (size & (size - 1)) == 0;
bool useLibCall = !powerOf2Size || (size > 16);
// For atomics larger than 16 bytes, emit a libcall from the frontend. This
// avoids the overhead of dealing with excessively-large value types in IR.
// Non-power-of-2 values also lower to libcall here, as they are not currently
// permitted in IR instructions (although that constraint could be relaxed in
// the future). For other cases where a libcall is required on a given
// platform, we let the backend handle it (this includes handling for all of
// the size-optimized libcall variants, which are only valid up to 16 bytes.)
//
// See: https://llvm.org/docs/Atomics.html#libcalls-atomic
if (useLibCall) {
assert(!cir::MissingFeatures::atomicUseLibCall());
cgm.errorNYI(e->getSourceRange(), "emitAtomicExpr: emit atomic lib call");
return RValue::get(nullptr);
}
bool isStore = e->getOp() == AtomicExpr::AO__c11_atomic_store ||
e->getOp() == AtomicExpr::AO__atomic_store;
bool isLoad = e->getOp() == AtomicExpr::AO__c11_atomic_load ||
e->getOp() == AtomicExpr::AO__atomic_load;
if (!order) {
// We have evaluated the memory order as an integer constant in orderConst.
// We should not ever get to a case where the ordering isn't a valid CABI
// value, but it's hard to enforce that in general.
uint64_t ord = orderConst.Val.getInt().getZExtValue();
if (isMemOrderValid(ord, isStore, isLoad))
emitAtomicOp(*this, e, dest, ptr, val1, size,
static_cast<cir::MemOrder>(ord));
} else {
assert(!cir::MissingFeatures::atomicExpr());
cgm.errorNYI(e->getSourceRange(), "emitAtomicExpr: dynamic memory order");
return RValue::get(nullptr);
}
if (resultTy->isVoidType())
return RValue::get(nullptr);
return convertTempToRValue(
dest.withElementType(builder, convertTypeForMem(resultTy)), resultTy,
e->getExprLoc());
}
void CIRGenFunction::emitAtomicInit(Expr *init, LValue dest) {

12
clang/lib/CIR/CodeGen/CIRGenBuilder.h
View File

@ -373,15 +373,19 @@ public:
cir::LoadOp createLoad(mlir::Location loc, Address addr,
bool isVolatile = false) {
mlir::IntegerAttr align = getAlignmentAttr(addr.getAlignment());
return create<cir::LoadOp>(loc, addr.getPointer(), /*isDeref=*/false,
align);
return cir::LoadOp::create(*this, loc, addr.getPointer(), /*isDeref=*/false,
/*alignment=*/align,
/*mem_order=*/cir::MemOrderAttr{});
}
cir::StoreOp createStore(mlir::Location loc, mlir::Value val, Address dst,
mlir::IntegerAttr align = {}) {
bool isVolatile = false,
mlir::IntegerAttr align = {},
cir::MemOrderAttr order = {}) {
if (!align)
align = getAlignmentAttr(dst.getAlignment());
return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), align);
return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), isVolatile,
align, order);
}
/// Create a cir.complex.real_ptr operation that derives a pointer to the real

2
clang/lib/CIR/CodeGen/CIRGenFunction.h
View File

@ -1446,7 +1446,7 @@ public:
mlir::OpBuilder::InsertionGuard guard(builder);
builder.restoreInsertionPoint(outermostConditional->getInsertPoint());
builder.createStore(
value.getLoc(), value, addr,
value.getLoc(), value, addr, /*isVolatile=*/false,
mlir::IntegerAttr::get(
mlir::IntegerType::get(value.getContext(), 64),
(uint64_t)addr.getAlignment().getAsAlign().value()));

42
clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
View File

@ -1165,12 +1165,33 @@ mlir::LogicalResult CIRToLLVMFrameAddrOpLowering::matchAndRewrite(
return mlir::success();
}
static mlir::LLVM::AtomicOrdering
getLLVMMemOrder(std::optional<cir::MemOrder> memorder) {
if (!memorder)
return mlir::LLVM::AtomicOrdering::not_atomic;
switch (*memorder) {
case cir::MemOrder::Relaxed:
return mlir::LLVM::AtomicOrdering::monotonic;
case cir::MemOrder::Consume:
case cir::MemOrder::Acquire:
return mlir::LLVM::AtomicOrdering::acquire;
case cir::MemOrder::Release:
return mlir::LLVM::AtomicOrdering::release;
case cir::MemOrder::AcquireRelease:
return mlir::LLVM::AtomicOrdering::acq_rel;
case cir::MemOrder::SequentiallyConsistent:
return mlir::LLVM::AtomicOrdering::seq_cst;
default:
llvm_unreachable("unknown memory order");
}
}
mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
cir::LoadOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type llvmTy =
convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
assert(!cir::MissingFeatures::opLoadStoreMemOrder());
mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(op.getMemOrder());
std::optional<size_t> opAlign = op.getAlignment();
unsigned alignment =
(unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
@ -1179,11 +1200,10 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
// TODO: nontemporal, syncscope.
assert(!cir::MissingFeatures::opLoadStoreVolatile());
mlir::LLVM::LoadOp newLoad = rewriter.create<mlir::LLVM::LoadOp>(
op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
/*volatile=*/false, /*nontemporal=*/false,
/*invariant=*/false, /*invariantGroup=*/false,
mlir::LLVM::AtomicOrdering::not_atomic);
mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(
rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
/*isVolatile=*/false, /*isNonTemporal=*/false,
/*isInvariant=*/false, /*isInvariantGroup=*/false, ordering);
// Convert adapted result to its original type if needed.
mlir::Value result =
@ -1196,7 +1216,7 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
cir::StoreOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
assert(!cir::MissingFeatures::opLoadStoreMemOrder());
mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(op.getMemOrder());
const mlir::Type llvmTy =
getTypeConverter()->convertType(op.getValue().getType());
std::optional<size_t> opAlign = op.getAlignment();
@ -1210,10 +1230,10 @@ mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
op.getValue().getType(), adaptor.getValue());
// TODO: nontemporal, syncscope.
assert(!cir::MissingFeatures::opLoadStoreVolatile());
mlir::LLVM::StoreOp storeOp = rewriter.create<mlir::LLVM::StoreOp>(
op->getLoc(), value, adaptor.getAddr(), alignment, /*volatile=*/false,
/*nontemporal=*/false, /*invariantGroup=*/false,
mlir::LLVM::AtomicOrdering::not_atomic);
mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(
rewriter, op->getLoc(), value, adaptor.getAddr(), alignment,
/*isVolatile=*/false,
/*isNonTemporal=*/false, /*isInvariantGroup=*/false, memorder);
rewriter.replaceOp(op, storeOp);
assert(!cir::MissingFeatures::opLoadStoreTbaa());
return mlir::LogicalResult::success();

106
clang/test/CIR/CodeGen/atomic.c
View File

@ -45,3 +45,109 @@ void f2(void) {
// OGCG: %[[SLOT:.+]] = alloca i32, align 4
// OGCG-NEXT: store i32 42, ptr %[[SLOT]], align 4
// OGCG: }
void load(int *ptr) {
int x;
__atomic_load(ptr, &x, __ATOMIC_RELAXED);
__atomic_load(ptr, &x, __ATOMIC_CONSUME);
__atomic_load(ptr, &x, __ATOMIC_ACQUIRE);
__atomic_load(ptr, &x, __ATOMIC_SEQ_CST);
}
// CIR-LABEL: @load
// CIR: %{{.+}} = cir.load align(4) atomic(relaxed) %{{.+}} : !cir.ptr<!s32i>, !s32i
// CIR: %{{.+}} = cir.load align(4) atomic(consume) %{{.+}} : !cir.ptr<!s32i>, !s32i
// CIR: %{{.+}} = cir.load align(4) atomic(acquire) %{{.+}} : !cir.ptr<!s32i>, !s32i
// CIR: %{{.+}} = cir.load align(4) atomic(seq_cst) %{{.+}} : !cir.ptr<!s32i>, !s32i
// CIR: }
// LLVM-LABEL: @load
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} seq_cst, align 4
// LLVM: }
// OGCG-LABEL: @load
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} seq_cst, align 4
// OGCG: }
void c11_load(_Atomic(int) *ptr) {
__c11_atomic_load(ptr, __ATOMIC_RELAXED);
__c11_atomic_load(ptr, __ATOMIC_CONSUME);
__c11_atomic_load(ptr, __ATOMIC_ACQUIRE);
__c11_atomic_load(ptr, __ATOMIC_SEQ_CST);
}
// CIR-LABEL: @c11_load
// CIR: %{{.+}} = cir.load align(4) atomic(relaxed) %{{.+}} : !cir.ptr<!s32i>, !s32i
// CIR: %{{.+}} = cir.load align(4) atomic(consume) %{{.+}} : !cir.ptr<!s32i>, !s32i
// CIR: %{{.+}} = cir.load align(4) atomic(acquire) %{{.+}} : !cir.ptr<!s32i>, !s32i
// CIR: %{{.+}} = cir.load align(4) atomic(seq_cst) %{{.+}} : !cir.ptr<!s32i>, !s32i
// CIR: }
// LLVM-LABEL: @c11_load
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} seq_cst, align 4
// LLVM: }
// OGCG-LABEL: @c11_load
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} seq_cst, align 4
// OGCG: }
void store(int *ptr, int x) {
__atomic_store(ptr, &x, __ATOMIC_RELAXED);
__atomic_store(ptr, &x, __ATOMIC_RELEASE);
__atomic_store(ptr, &x, __ATOMIC_SEQ_CST);
}
// CIR-LABEL: @store
// CIR: cir.store align(4) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
// CIR: cir.store align(4) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
// CIR: cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
// CIR: }
// LLVM-LABEL: @store
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} release, align 4
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} seq_cst, align 4
// LLVM: }
// OGCG-LABEL: @store
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} release, align 4
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} seq_cst, align 4
// OGCG: }
void c11_store(_Atomic(int) *ptr, int x) {
__c11_atomic_store(ptr, x, __ATOMIC_RELAXED);
__c11_atomic_store(ptr, x, __ATOMIC_RELEASE);
__c11_atomic_store(ptr, x, __ATOMIC_SEQ_CST);
}
// CIR-LABEL: @c11_store
// CIR: cir.store align(4) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
// CIR: cir.store align(4) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
// CIR: cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
// CIR: }
// LLVM-LABEL: @c11_store
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} release, align 4
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} seq_cst, align 4
// LLVM: }
// OGCG-LABEL: @c11_store
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} release, align 4
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} seq_cst, align 4
// OGCG: }