58a88001f3
This fixes the handling of subregister extract copies. This will allow AMDGPU to remove its implementation of shouldRewriteCopySrc, which exists as a 10 year old workaround to this bug. peephole-opt-fold-reg-sequence-subreg.mir will show the expected improvement once the custom implementation is removed. The copy coalescing processing here is overly abstracted from what's actually happening. Previously when visiting coalescable copy-like instructions, we would parse the sources one at a time and then pass the def of the root instruction into findNextSource. This means that the first thing the new ValueTracker constructed would do is getVRegDef to find the instruction we are currently processing. This adds an unnecessary step, placing a useless entry in the RewriteMap, and required skipping the no-op case where getNewSource would return the original source operand. This was a problem since in the case of a subregister extract, shouldRewriteCopySource would always say that it is useful to rewrite and the use-def chain walk would abort, returning the original operand. Move the process to start looking at the source operand to begin with. This does not fix the confused handling in the uncoalescable copy case which is proving to be more difficult. Some currently handled cases have multiple defs from a single source, and other handled cases have 0 input operands. It would be simpler if this was implemented with isCopyLikeInstr, rather than guessing at the operand structure as it does now. There are some improvements and some regressions. The regressions appear to be downstream issues for the most part. One of the uglier regressions is in PPC, where a sequence of insert_subrgs is used to build registers. I opened #125502 to use reg_sequence instead, which may help. The worst regression is an absurd SPARC testcase using a <251 x fp128>, which uses a very long chain of insert_subregs. We need improved subregister handling locally in PeepholeOptimizer, and other pasess like MachineCSE to fix some of the other regressions. We should handle subregister composes and folding more indexes into insert_subreg and reg_sequence.
1270 lines
49 KiB
LLVM
1270 lines
49 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -aarch64-enable-sink-fold=true < %s | FileCheck %s --check-prefixes=CHECK,INLINE_ATOMICS
|
|
; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+outline-atomics -aarch64-enable-sink-fold=true < %s | FileCheck %s --check-prefixes=CHECK,OUTLINE_ATOMICS
|
|
|
|
@var8 = dso_local global i8 0
|
|
@var16 = dso_local global i16 0
|
|
@var32 = dso_local global i32 0
|
|
@var64 = dso_local global i64 0
|
|
|
|
define dso_local i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_add_i8:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var8
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
|
; INLINE_ATOMICS-NEXT: .LBB0_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxrb w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: add w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB0_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i8:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw add ptr @var8, i8 %offset seq_cst
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_add_i16:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var16
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
|
; INLINE_ATOMICS-NEXT: .LBB1_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxrh w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: add w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB1_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i16:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd2_acq
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw add ptr @var16, i16 %offset acquire
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_add_i32:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var32
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
|
; INLINE_ATOMICS-NEXT: .LBB2_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxr w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: add w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB2_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i32:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd4_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw add ptr @var32, i32 %offset release
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_add_i64:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var64
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
|
; INLINE_ATOMICS-NEXT: .LBB3_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxr x8, [x9]
|
|
; INLINE_ATOMICS-NEXT: add x10, x8, x0
|
|
; INLINE_ATOMICS-NEXT: stxr w11, x10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB3_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov x0, x8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i64:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd8_relax
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw add ptr @var64, i64 %offset monotonic
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i8:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var8
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
|
; INLINE_ATOMICS-NEXT: .LBB4_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxrb w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: sub w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stxrb w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB4_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i8:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: neg w0, w0
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd1_relax
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw sub ptr @var8, i8 %offset monotonic
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i16:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var16
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
|
; INLINE_ATOMICS-NEXT: .LBB5_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxrh w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: sub w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stlxrh w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB5_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i16:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: neg w0, w0
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd2_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw sub ptr @var16, i16 %offset release
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i32:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var32
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
|
; INLINE_ATOMICS-NEXT: .LBB6_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxr w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: sub w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stxr w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB6_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i32:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: neg w0, w0
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd4_acq
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw sub ptr @var32, i32 %offset acquire
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i64:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var64
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
|
; INLINE_ATOMICS-NEXT: .LBB7_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxr x8, [x9]
|
|
; INLINE_ATOMICS-NEXT: sub x10, x8, x0
|
|
; INLINE_ATOMICS-NEXT: stlxr w11, x10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB7_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov x0, x8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i64:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: neg x0, x0
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw sub ptr @var64, i64 %offset seq_cst
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_and_i8:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var8
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
|
; INLINE_ATOMICS-NEXT: .LBB8_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxrb w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: and w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB8_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i8:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
|
; OUTLINE_ATOMICS-NEXT: mvn w0, w0
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr1_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw and ptr @var8, i8 %offset release
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_and_i16:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var16
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
|
; INLINE_ATOMICS-NEXT: .LBB9_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxrh w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: and w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB9_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i16:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
|
; OUTLINE_ATOMICS-NEXT: mvn w0, w0
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr2_relax
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw and ptr @var16, i16 %offset monotonic
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_and_i32:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var32
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
|
; INLINE_ATOMICS-NEXT: .LBB10_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxr w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: and w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB10_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i32:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
|
; OUTLINE_ATOMICS-NEXT: mvn w0, w0
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw and ptr @var32, i32 %offset seq_cst
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_and_i64:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var64
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
|
; INLINE_ATOMICS-NEXT: .LBB11_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxr x8, [x9]
|
|
; INLINE_ATOMICS-NEXT: and x10, x8, x0
|
|
; INLINE_ATOMICS-NEXT: stxr w11, x10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB11_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov x0, x8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i64:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
|
; OUTLINE_ATOMICS-NEXT: mvn x0, x0
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr8_acq
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw and ptr @var64, i64 %offset acquire
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_or_i8:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var8
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
|
; INLINE_ATOMICS-NEXT: .LBB12_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxrb w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: orr w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB12_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i8:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw or ptr @var8, i8 %offset seq_cst
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_or_i16:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var16
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
|
; INLINE_ATOMICS-NEXT: .LBB13_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxrh w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: orr w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB13_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i16:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset2_relax
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw or ptr @var16, i16 %offset monotonic
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_or_i32:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var32
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
|
; INLINE_ATOMICS-NEXT: .LBB14_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxr w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: orr w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stxr w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB14_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i32:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset4_acq
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw or ptr @var32, i32 %offset acquire
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_or_i64:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var64
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
|
; INLINE_ATOMICS-NEXT: .LBB15_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxr x8, [x9]
|
|
; INLINE_ATOMICS-NEXT: orr x10, x8, x0
|
|
; INLINE_ATOMICS-NEXT: stlxr w11, x10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB15_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov x0, x8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i64:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset8_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw or ptr @var64, i64 %offset release
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i8:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var8
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
|
; INLINE_ATOMICS-NEXT: .LBB16_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxrb w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: eor w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stxrb w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB16_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i8:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor1_acq
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw xor ptr @var8, i8 %offset acquire
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i16:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var16
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
|
; INLINE_ATOMICS-NEXT: .LBB17_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxrh w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: eor w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stlxrh w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB17_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i16:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor2_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw xor ptr @var16, i16 %offset release
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i32:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var32
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
|
; INLINE_ATOMICS-NEXT: .LBB18_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxr w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: eor w10, w8, w0
|
|
; INLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB18_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i32:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw xor ptr @var32, i32 %offset seq_cst
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i64:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var64
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
|
; INLINE_ATOMICS-NEXT: .LBB19_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxr x8, [x9]
|
|
; INLINE_ATOMICS-NEXT: eor x10, x8, x0
|
|
; INLINE_ATOMICS-NEXT: stxr w11, x10, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w11, .LBB19_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov x0, x8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i64:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor8_relax
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw xor ptr @var64, i64 %offset monotonic
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var8
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
|
; INLINE_ATOMICS-NEXT: .LBB20_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxrb w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: stxrb w10, w0, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w10, .LBB20_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp1_relax
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw xchg ptr @var8, i8 %offset monotonic
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var16
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
|
; INLINE_ATOMICS-NEXT: .LBB21_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxrh w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: stlxrh w10, w0, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w10, .LBB21_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov w0, w8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp2_acq_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw xchg ptr @var16, i16 %offset seq_cst
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: mov w8, w0
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var32
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
|
; INLINE_ATOMICS-NEXT: .LBB22_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxr w0, [x9]
|
|
; INLINE_ATOMICS-NEXT: stlxr w10, w8, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w10, .LBB22_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp4_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw xchg ptr @var32, i32 %offset release
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var64
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
|
; INLINE_ATOMICS-NEXT: .LBB23_1: // %atomicrmw.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxr x8, [x9]
|
|
; INLINE_ATOMICS-NEXT: stxr w10, x0, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w10, .LBB23_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
|
; INLINE_ATOMICS-NEXT: mov x0, x8
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
|
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp8_acq
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%old = atomicrmw xchg ptr @var64, i64 %offset acquire
|
|
ret i64 %old
|
|
}
|
|
|
|
|
|
define dso_local i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_min_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var8
|
|
; CHECK-NEXT: add x9, x9, :lo12:var8
|
|
; CHECK-NEXT: .LBB24_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxrb w10, [x9]
|
|
; CHECK-NEXT: sxtb w8, w10
|
|
; CHECK-NEXT: cmp w8, w0, sxtb
|
|
; CHECK-NEXT: csel w10, w10, w0, le
|
|
; CHECK-NEXT: stxrb w11, w10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB24_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov w0, w8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw min ptr @var8, i8 %offset acquire
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_min_i16:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var16
|
|
; CHECK-NEXT: add x9, x9, :lo12:var16
|
|
; CHECK-NEXT: .LBB25_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldxrh w10, [x9]
|
|
; CHECK-NEXT: sxth w8, w10
|
|
; CHECK-NEXT: cmp w8, w0, sxth
|
|
; CHECK-NEXT: csel w10, w10, w0, le
|
|
; CHECK-NEXT: stlxrh w11, w10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB25_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov w0, w8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw min ptr @var16, i16 %offset release
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_min_i32:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var32
|
|
; CHECK-NEXT: add x9, x9, :lo12:var32
|
|
; CHECK-NEXT: .LBB26_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldxr w8, [x9]
|
|
; CHECK-NEXT: cmp w8, w0
|
|
; CHECK-NEXT: csel w10, w8, w0, le
|
|
; CHECK-NEXT: stxr w11, w10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB26_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov w0, w8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw min ptr @var32, i32 %offset monotonic
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_min_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var64
|
|
; CHECK-NEXT: add x9, x9, :lo12:var64
|
|
; CHECK-NEXT: .LBB27_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxr x8, [x9]
|
|
; CHECK-NEXT: cmp x8, x0
|
|
; CHECK-NEXT: csel x10, x8, x0, le
|
|
; CHECK-NEXT: stlxr w11, x10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB27_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov x0, x8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw min ptr @var64, i64 %offset seq_cst
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_max_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var8
|
|
; CHECK-NEXT: add x9, x9, :lo12:var8
|
|
; CHECK-NEXT: .LBB28_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxrb w10, [x9]
|
|
; CHECK-NEXT: sxtb w8, w10
|
|
; CHECK-NEXT: cmp w8, w0, sxtb
|
|
; CHECK-NEXT: csel w10, w10, w0, gt
|
|
; CHECK-NEXT: stlxrb w11, w10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB28_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov w0, w8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw max ptr @var8, i8 %offset seq_cst
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_max_i16:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var16
|
|
; CHECK-NEXT: add x9, x9, :lo12:var16
|
|
; CHECK-NEXT: .LBB29_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxrh w10, [x9]
|
|
; CHECK-NEXT: sxth w8, w10
|
|
; CHECK-NEXT: cmp w8, w0, sxth
|
|
; CHECK-NEXT: csel w10, w10, w0, gt
|
|
; CHECK-NEXT: stxrh w11, w10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB29_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov w0, w8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw max ptr @var16, i16 %offset acquire
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_max_i32:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var32
|
|
; CHECK-NEXT: add x9, x9, :lo12:var32
|
|
; CHECK-NEXT: .LBB30_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldxr w8, [x9]
|
|
; CHECK-NEXT: cmp w8, w0
|
|
; CHECK-NEXT: csel w10, w8, w0, gt
|
|
; CHECK-NEXT: stlxr w11, w10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB30_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov w0, w8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw max ptr @var32, i32 %offset release
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_max_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var64
|
|
; CHECK-NEXT: add x9, x9, :lo12:var64
|
|
; CHECK-NEXT: .LBB31_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldxr x8, [x9]
|
|
; CHECK-NEXT: cmp x8, x0
|
|
; CHECK-NEXT: csel x10, x8, x0, gt
|
|
; CHECK-NEXT: stxr w11, x10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB31_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov x0, x8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw max ptr @var64, i64 %offset monotonic
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_umin_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var8
|
|
; CHECK-NEXT: add x8, x8, :lo12:var8
|
|
; CHECK-NEXT: and w9, w0, #0xff
|
|
; CHECK-NEXT: .LBB32_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldxrb w0, [x8]
|
|
; CHECK-NEXT: cmp w0, w9
|
|
; CHECK-NEXT: csel w10, w0, w9, ls
|
|
; CHECK-NEXT: stxrb w11, w10, [x8]
|
|
; CHECK-NEXT: cbnz w11, .LBB32_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw umin ptr @var8, i8 %offset monotonic
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_umin_i16:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var16
|
|
; CHECK-NEXT: add x8, x8, :lo12:var16
|
|
; CHECK-NEXT: and w9, w0, #0xffff
|
|
; CHECK-NEXT: .LBB33_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxrh w0, [x8]
|
|
; CHECK-NEXT: cmp w0, w9
|
|
; CHECK-NEXT: csel w10, w0, w9, ls
|
|
; CHECK-NEXT: stxrh w11, w10, [x8]
|
|
; CHECK-NEXT: cbnz w11, .LBB33_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw umin ptr @var16, i16 %offset acquire
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_umin_i32:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var32
|
|
; CHECK-NEXT: add x9, x9, :lo12:var32
|
|
; CHECK-NEXT: .LBB34_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxr w8, [x9]
|
|
; CHECK-NEXT: cmp w8, w0
|
|
; CHECK-NEXT: csel w10, w8, w0, ls
|
|
; CHECK-NEXT: stlxr w11, w10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB34_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov w0, w8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw umin ptr @var32, i32 %offset seq_cst
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_umin_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var64
|
|
; CHECK-NEXT: add x9, x9, :lo12:var64
|
|
; CHECK-NEXT: .LBB35_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxr x8, [x9]
|
|
; CHECK-NEXT: cmp x8, x0
|
|
; CHECK-NEXT: csel x10, x8, x0, ls
|
|
; CHECK-NEXT: stlxr w11, x10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB35_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov x0, x8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw umin ptr @var64, i64 %offset acq_rel
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_umax_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var8
|
|
; CHECK-NEXT: add x8, x8, :lo12:var8
|
|
; CHECK-NEXT: and w9, w0, #0xff
|
|
; CHECK-NEXT: .LBB36_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxrb w0, [x8]
|
|
; CHECK-NEXT: cmp w0, w9
|
|
; CHECK-NEXT: csel w10, w0, w9, hi
|
|
; CHECK-NEXT: stlxrb w11, w10, [x8]
|
|
; CHECK-NEXT: cbnz w11, .LBB36_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw umax ptr @var8, i8 %offset acq_rel
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_umax_i16:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var16
|
|
; CHECK-NEXT: add x8, x8, :lo12:var16
|
|
; CHECK-NEXT: and w9, w0, #0xffff
|
|
; CHECK-NEXT: .LBB37_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldxrh w0, [x8]
|
|
; CHECK-NEXT: cmp w0, w9
|
|
; CHECK-NEXT: csel w10, w0, w9, hi
|
|
; CHECK-NEXT: stxrh w11, w10, [x8]
|
|
; CHECK-NEXT: cbnz w11, .LBB37_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw umax ptr @var16, i16 %offset monotonic
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_umax_i32:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var32
|
|
; CHECK-NEXT: add x9, x9, :lo12:var32
|
|
; CHECK-NEXT: .LBB38_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldaxr w8, [x9]
|
|
; CHECK-NEXT: cmp w8, w0
|
|
; CHECK-NEXT: csel w10, w8, w0, hi
|
|
; CHECK-NEXT: stlxr w11, w10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB38_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov w0, w8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw umax ptr @var32, i32 %offset seq_cst
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_umax_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x9, var64
|
|
; CHECK-NEXT: add x9, x9, :lo12:var64
|
|
; CHECK-NEXT: .LBB39_1: // %atomicrmw.start
|
|
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: ldxr x8, [x9]
|
|
; CHECK-NEXT: cmp x8, x0
|
|
; CHECK-NEXT: csel x10, x8, x0, hi
|
|
; CHECK-NEXT: stlxr w11, x10, [x9]
|
|
; CHECK-NEXT: cbnz w11, .LBB39_1
|
|
; CHECK-NEXT: // %bb.2: // %atomicrmw.end
|
|
; CHECK-NEXT: mov x0, x8
|
|
; CHECK-NEXT: ret
|
|
%old = atomicrmw umax ptr @var64, i64 %offset release
|
|
ret i64 %old
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: and w8, w0, #0xff
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var8
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
|
; INLINE_ATOMICS-NEXT: .LBB40_1: // %cmpxchg.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxrb w0, [x9]
|
|
; INLINE_ATOMICS-NEXT: cmp w0, w8
|
|
; INLINE_ATOMICS-NEXT: b.ne .LBB40_4
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %cmpxchg.trystore
|
|
; INLINE_ATOMICS-NEXT: // in Loop: Header=BB40_1 Depth=1
|
|
; INLINE_ATOMICS-NEXT: stxrb w10, w1, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w10, .LBB40_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.3: // %cmpxchg.end
|
|
; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
; INLINE_ATOMICS-NEXT: .LBB40_4: // %cmpxchg.nostore
|
|
; INLINE_ATOMICS-NEXT: clrex
|
|
; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x2, var8
|
|
; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var8
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas1_acq
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%pair = cmpxchg ptr @var8, i8 %wanted, i8 %new acquire acquire
|
|
%old = extractvalue { i8, i1 } %pair, 0
|
|
ret i8 %old
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: and w8, w0, #0xffff
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var16
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
|
; INLINE_ATOMICS-NEXT: .LBB41_1: // %cmpxchg.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldaxrh w0, [x9]
|
|
; INLINE_ATOMICS-NEXT: cmp w0, w8
|
|
; INLINE_ATOMICS-NEXT: b.ne .LBB41_4
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %cmpxchg.trystore
|
|
; INLINE_ATOMICS-NEXT: // in Loop: Header=BB41_1 Depth=1
|
|
; INLINE_ATOMICS-NEXT: stlxrh w10, w1, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w10, .LBB41_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.3: // %cmpxchg.end
|
|
; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
; INLINE_ATOMICS-NEXT: .LBB41_4: // %cmpxchg.nostore
|
|
; INLINE_ATOMICS-NEXT: clrex
|
|
; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x2, var16
|
|
; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var16
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas2_acq_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%pair = cmpxchg ptr @var16, i16 %wanted, i16 %new seq_cst seq_cst
|
|
%old = extractvalue { i16, i1 } %pair, 0
|
|
ret i16 %old
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: mov w8, w0
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var32
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
|
; INLINE_ATOMICS-NEXT: .LBB42_1: // %cmpxchg.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxr w0, [x9]
|
|
; INLINE_ATOMICS-NEXT: cmp w0, w8
|
|
; INLINE_ATOMICS-NEXT: b.ne .LBB42_4
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %cmpxchg.trystore
|
|
; INLINE_ATOMICS-NEXT: // in Loop: Header=BB42_1 Depth=1
|
|
; INLINE_ATOMICS-NEXT: stlxr w10, w1, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w10, .LBB42_1
|
|
; INLINE_ATOMICS-NEXT: // %bb.3: // %cmpxchg.end
|
|
; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
; INLINE_ATOMICS-NEXT: .LBB42_4: // %cmpxchg.nostore
|
|
; INLINE_ATOMICS-NEXT: clrex
|
|
; INLINE_ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x2, var32
|
|
; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var32
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas4_rel
|
|
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%pair = cmpxchg ptr @var32, i32 %wanted, i32 %new release monotonic
|
|
%old = extractvalue { i32, i1 } %pair, 0
|
|
ret i32 %old
|
|
}
|
|
|
|
define dso_local void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
|
|
; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64:
|
|
; INLINE_ATOMICS: // %bb.0:
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var64
|
|
; INLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
|
; INLINE_ATOMICS-NEXT: .LBB43_1: // %cmpxchg.start
|
|
; INLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
|
; INLINE_ATOMICS-NEXT: ldxr x8, [x9]
|
|
; INLINE_ATOMICS-NEXT: cmp x8, x0
|
|
; INLINE_ATOMICS-NEXT: b.ne .LBB43_3
|
|
; INLINE_ATOMICS-NEXT: // %bb.2: // %cmpxchg.trystore
|
|
; INLINE_ATOMICS-NEXT: // in Loop: Header=BB43_1 Depth=1
|
|
; INLINE_ATOMICS-NEXT: stxr w10, x1, [x9]
|
|
; INLINE_ATOMICS-NEXT: cbnz w10, .LBB43_1
|
|
; INLINE_ATOMICS-NEXT: b .LBB43_4
|
|
; INLINE_ATOMICS-NEXT: .LBB43_3: // %cmpxchg.nostore
|
|
; INLINE_ATOMICS-NEXT: clrex
|
|
; INLINE_ATOMICS-NEXT: .LBB43_4: // %cmpxchg.end
|
|
; INLINE_ATOMICS-NEXT: adrp x9, var64
|
|
; INLINE_ATOMICS-NEXT: str x8, [x9, :lo12:var64]
|
|
; INLINE_ATOMICS-NEXT: ret
|
|
;
|
|
; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64:
|
|
; OUTLINE_ATOMICS: // %bb.0:
|
|
; OUTLINE_ATOMICS-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
|
|
; OUTLINE_ATOMICS-NEXT: adrp x19, var64
|
|
; OUTLINE_ATOMICS-NEXT: add x19, x19, :lo12:var64
|
|
; OUTLINE_ATOMICS-NEXT: mov x2, x19
|
|
; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas8_relax
|
|
; OUTLINE_ATOMICS-NEXT: str x0, [x19]
|
|
; OUTLINE_ATOMICS-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload
|
|
; OUTLINE_ATOMICS-NEXT: ret
|
|
%pair = cmpxchg ptr @var64, i64 %wanted, i64 %new monotonic monotonic
|
|
%old = extractvalue { i64, i1 } %pair, 0
|
|
store i64 %old, ptr @var64
|
|
ret void
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_monotonic_i8() nounwind {
|
|
; CHECK-LABEL: test_atomic_load_monotonic_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var8
|
|
; CHECK-NEXT: ldrb w0, [x8, :lo12:var8]
|
|
; CHECK-NEXT: ret
|
|
%val = load atomic i8, ptr @var8 monotonic, align 1
|
|
ret i8 %val
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: ldrb w0, [x0, x1]
|
|
; CHECK-NEXT: ret
|
|
%addr_int = add i64 %base, %off
|
|
%addr = inttoptr i64 %addr_int to ptr
|
|
%val = load atomic i8, ptr %addr monotonic, align 1
|
|
ret i8 %val
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_acquire_i8() nounwind {
|
|
; CHECK-LABEL: test_atomic_load_acquire_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var8
|
|
; CHECK-NEXT: add x8, x8, :lo12:var8
|
|
; CHECK-NEXT: ldarb w0, [x8]
|
|
; CHECK-NEXT: ret
|
|
%val = load atomic i8, ptr @var8 acquire, align 1
|
|
ret i8 %val
|
|
}
|
|
|
|
define dso_local i8 @test_atomic_load_seq_cst_i8() nounwind {
|
|
; CHECK-LABEL: test_atomic_load_seq_cst_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var8
|
|
; CHECK-NEXT: add x8, x8, :lo12:var8
|
|
; CHECK-NEXT: ldarb w0, [x8]
|
|
; CHECK-NEXT: ret
|
|
%val = load atomic i8, ptr @var8 seq_cst, align 1
|
|
ret i8 %val
|
|
}
|
|
|
|
define dso_local i16 @test_atomic_load_monotonic_i16() nounwind {
|
|
; CHECK-LABEL: test_atomic_load_monotonic_i16:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var16
|
|
; CHECK-NEXT: ldrh w0, [x8, :lo12:var16]
|
|
; CHECK-NEXT: ret
|
|
%val = load atomic i16, ptr @var16 monotonic, align 2
|
|
ret i16 %val
|
|
}
|
|
|
|
define dso_local i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
|
|
; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: ldr w0, [x0, x1]
|
|
; CHECK-NEXT: ret
|
|
%addr_int = add i64 %base, %off
|
|
%addr = inttoptr i64 %addr_int to ptr
|
|
%val = load atomic i32, ptr %addr monotonic, align 4
|
|
ret i32 %val
|
|
}
|
|
|
|
define dso_local i64 @test_atomic_load_seq_cst_i64() nounwind {
|
|
; CHECK-LABEL: test_atomic_load_seq_cst_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var64
|
|
; CHECK-NEXT: add x8, x8, :lo12:var64
|
|
; CHECK-NEXT: ldar x0, [x8]
|
|
; CHECK-NEXT: ret
|
|
%val = load atomic i64, ptr @var64 seq_cst, align 8
|
|
ret i64 %val
|
|
}
|
|
|
|
define dso_local void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
|
|
; CHECK-LABEL: test_atomic_store_monotonic_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var8
|
|
; CHECK-NEXT: strb w0, [x8, :lo12:var8]
|
|
; CHECK-NEXT: ret
|
|
store atomic i8 %val, ptr @var8 monotonic, align 1
|
|
ret void
|
|
}
|
|
|
|
define dso_local void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
|
|
; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: strb w2, [x0, x1]
|
|
; CHECK-NEXT: ret
|
|
%addr_int = add i64 %base, %off
|
|
%addr = inttoptr i64 %addr_int to ptr
|
|
store atomic i8 %val, ptr %addr monotonic, align 1
|
|
ret void
|
|
}
|
|
define dso_local void @test_atomic_store_release_i8(i8 %val) nounwind {
|
|
; CHECK-LABEL: test_atomic_store_release_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var8
|
|
; CHECK-NEXT: add x8, x8, :lo12:var8
|
|
; CHECK-NEXT: stlrb w0, [x8]
|
|
; CHECK-NEXT: ret
|
|
store atomic i8 %val, ptr @var8 release, align 1
|
|
ret void
|
|
}
|
|
|
|
define dso_local void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
|
|
; CHECK-LABEL: test_atomic_store_seq_cst_i8:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var8
|
|
; CHECK-NEXT: add x8, x8, :lo12:var8
|
|
; CHECK-NEXT: stlrb w0, [x8]
|
|
; CHECK-NEXT: ret
|
|
store atomic i8 %val, ptr @var8 seq_cst, align 1
|
|
ret void
|
|
}
|
|
|
|
define dso_local void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
|
|
; CHECK-LABEL: test_atomic_store_monotonic_i16:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var16
|
|
; CHECK-NEXT: strh w0, [x8, :lo12:var16]
|
|
; CHECK-NEXT: ret
|
|
store atomic i16 %val, ptr @var16 monotonic, align 2
|
|
ret void
|
|
}
|
|
|
|
define dso_local void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
|
|
; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: str w2, [x0, x1]
|
|
; CHECK-NEXT: ret
|
|
%addr_int = add i64 %base, %off
|
|
%addr = inttoptr i64 %addr_int to ptr
|
|
store atomic i32 %val, ptr %addr monotonic, align 4
|
|
ret void
|
|
}
|
|
|
|
define dso_local void @test_atomic_store_release_i64(i64 %val) nounwind {
|
|
; CHECK-LABEL: test_atomic_store_release_i64:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: adrp x8, var64
|
|
; CHECK-NEXT: add x8, x8, :lo12:var64
|
|
; CHECK-NEXT: stlr x0, [x8]
|
|
; CHECK-NEXT: ret
|
|
store atomic i64 %val, ptr @var64 release, align 8
|
|
ret void
|
|
}
|