shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/test/CodeGen/AArch64/streaming-compatible-memory-ops.ll
Sander de Smalen 91f5d73b31
[AArch64] Add new pass after VirtRegRewriter to add implicit-defs (#174188) 
When SubRegister Liveness Tracking (SRLT) is enabled, this pass adds
extra implicit-def's to instructions that define the low N bits of a
GPR/FPR register to represent that the top bits are written, because all
AArch64 instructions that write the low bits of a GPR/FPR also
implicitly zero the top bits.

These semantics are originally represented in the MIR using
`SUBREG_TO_REG`, but during register coalescing this information is lost
and when rewriting virtual -> physical registers the implicit-defs are
not added to represent the the top bits are written.

There have been several attempts to fix this in the coalescer (#168353),
but each iteration has exposed new bugs and the patch had to be
reverted. Additionally, the concept of adding 'implicit-def' of a
virtual register during the register allocation process is particularly
fragile and many places don't expect it (for example in
`X86::commuteInstructionImpl` the code only looks at specific operands
and does not consider implicit-defs. Similar in
`SplitEditor::addDeadDef` where it traverses operand 'defs' rather than
'all_defs').

We want a temporary solution that doesn't impact other targets and is
simpler and less intrusive than the patch proposed for the register
coalescer so that we can enable SRLT to make better use of SVE/SME
multi-vector instructions while we work on a more permanent solution
that requires rewriting a large part of the AArch64 instructions (32-bit
and NEON).
2026-01-14 16:42:15 +00:00

453 lines
22 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK-COMMON,CHECK
; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -verify-machineinstrs -aarch64-lower-to-sme-routines=false < %s | FileCheck %s -check-prefixes=CHECK-COMMON,CHECK-NO-SME-ROUTINES
; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -mattr=+mops -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK-COMMON,CHECK-MOPS
@dst = global [512 x i8] zeroinitializer, align 1
@src = global [512 x i8] zeroinitializer, align 1
define void @se_memcpy(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {
; CHECK-LABEL: se_memcpy:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: mov x2, x0
; CHECK-NEXT: adrp x0, :got:dst
; CHECK-NEXT: adrp x1, :got:src
; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NEXT: ldr x1, [x1, :got_lo12:src]
; CHECK-NEXT: bl __arm_sc_memcpy
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
;
; CHECK-NO-SME-ROUTINES-LABEL: se_memcpy:
; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x0
; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst
; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: adrp x1, :got:src
; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: str x30, [sp, #64] // 8-byte Spill
; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NO-SME-ROUTINES-NEXT: ldr x1, [x1, :got_lo12:src]
; CHECK-NO-SME-ROUTINES-NEXT: smstop sm
; CHECK-NO-SME-ROUTINES-NEXT: bl memcpy
; CHECK-NO-SME-ROUTINES-NEXT: smstart sm
; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ret
;
; CHECK-MOPS-LABEL: se_memcpy:
; CHECK-MOPS: // %bb.0: // %entry
; CHECK-MOPS-NEXT: adrp x8, :got:src
; CHECK-MOPS-NEXT: adrp x9, :got:dst
; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:src]
; CHECK-MOPS-NEXT: ldr x9, [x9, :got_lo12:dst]
; CHECK-MOPS-NEXT: cpyfp [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: cpyfm [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: cpyfe [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: ret
entry:
tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
ret void
}
define void @se_memset(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {
; CHECK-LABEL: se_memset:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: mov x2, x0
; CHECK-NEXT: adrp x0, :got:dst
; CHECK-NEXT: mov w1, #2 // =0x2
; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NEXT: bl __arm_sc_memset
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
;
; CHECK-NO-SME-ROUTINES-LABEL: se_memset:
; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x0
; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst
; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: str x30, [sp, #64] // 8-byte Spill
; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NO-SME-ROUTINES-NEXT: smstop sm
; CHECK-NO-SME-ROUTINES-NEXT: mov w1, #2 // =0x2
; CHECK-NO-SME-ROUTINES-NEXT: bl memset
; CHECK-NO-SME-ROUTINES-NEXT: smstart sm
; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ret
;
; CHECK-MOPS-LABEL: se_memset:
; CHECK-MOPS: // %bb.0: // %entry
; CHECK-MOPS-NEXT: adrp x8, :got:dst
; CHECK-MOPS-NEXT: mov w9, #2 // =0x2
; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:dst]
; CHECK-MOPS-NEXT: setp [x8]!, x0!, x9
; CHECK-MOPS-NEXT: setm [x8]!, x0!, x9
; CHECK-MOPS-NEXT: sete [x8]!, x0!, x9
; CHECK-MOPS-NEXT: ret
entry:
tail call void @llvm.memset.p0.i64(ptr align 1 @dst, i8 2, i64 %n, i1 false)
ret void
}
define void @se_memmove(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {
; CHECK-LABEL: se_memmove:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: mov x2, x0
; CHECK-NEXT: adrp x0, :got:dst
; CHECK-NEXT: adrp x1, :got:src
; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NEXT: ldr x1, [x1, :got_lo12:src]
; CHECK-NEXT: bl __arm_sc_memmove
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
;
; CHECK-NO-SME-ROUTINES-LABEL: se_memmove:
; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x0
; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst
; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: adrp x1, :got:src
; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: str x30, [sp, #64] // 8-byte Spill
; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NO-SME-ROUTINES-NEXT: ldr x1, [x1, :got_lo12:src]
; CHECK-NO-SME-ROUTINES-NEXT: smstop sm
; CHECK-NO-SME-ROUTINES-NEXT: bl memmove
; CHECK-NO-SME-ROUTINES-NEXT: smstart sm
; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ret
;
; CHECK-MOPS-LABEL: se_memmove:
; CHECK-MOPS: // %bb.0: // %entry
; CHECK-MOPS-NEXT: adrp x8, :got:src
; CHECK-MOPS-NEXT: adrp x9, :got:dst
; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:src]
; CHECK-MOPS-NEXT: ldr x9, [x9, :got_lo12:dst]
; CHECK-MOPS-NEXT: cpyp [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: cpym [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: cpye [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: ret
entry:
tail call void @llvm.memmove.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
ret void
}
define ptr @se_memchr(ptr %src, i64 %n) "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: se_memchr:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: .cfi_offset w30, -16
; CHECK-NEXT: mov x2, x1
; CHECK-NEXT: mov w1, #5 // =0x5
; CHECK-NEXT: bl __arm_sc_memchr
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
;
; CHECK-NO-SME-ROUTINES-LABEL: se_memchr:
; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_def_cfa_offset 96
; CHECK-NO-SME-ROUTINES-NEXT: cntd x9
; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: str x9, [sp, #80] // 8-byte Spill
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset vg, -16
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset w30, -24
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset w29, -32
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b8, -40
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b9, -48
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b10, -56
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b11, -64
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b12, -72
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b13, -80
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b14, -88
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b15, -96
; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x1
; CHECK-NO-SME-ROUTINES-NEXT: smstop sm
; CHECK-NO-SME-ROUTINES-NEXT: mov w1, #5 // =0x5
; CHECK-NO-SME-ROUTINES-NEXT: bl memchr
; CHECK-NO-SME-ROUTINES-NEXT: smstart sm
; CHECK-NO-SME-ROUTINES-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_def_cfa_offset 0
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore vg
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore w30
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore w29
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b8
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b9
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b10
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b11
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b12
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b13
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b14
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b15
; CHECK-NO-SME-ROUTINES-NEXT: ret
;
; CHECK-MOPS-LABEL: se_memchr:
; CHECK-MOPS: // %bb.0: // %entry
; CHECK-MOPS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-MOPS-NEXT: .cfi_def_cfa_offset 16
; CHECK-MOPS-NEXT: .cfi_offset w30, -16
; CHECK-MOPS-NEXT: mov x2, x1
; CHECK-MOPS-NEXT: mov w1, #5 // =0x5
; CHECK-MOPS-NEXT: bl __arm_sc_memchr
; CHECK-MOPS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-MOPS-NEXT: ret
entry:
%res = tail call ptr @memchr(ptr %src, i32 5, i64 %n)
ret ptr %res
}
define ptr @sc_memchr(ptr %src, i64 %n) "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: sc_memchr:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: .cfi_offset w30, -16
; CHECK-NEXT: mov x2, x1
; CHECK-NEXT: mov w1, #5 // =0x5
; CHECK-NEXT: bl __arm_sc_memchr
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
;
; CHECK-NO-SME-ROUTINES-LABEL: sc_memchr:
; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_def_cfa_offset 96
; CHECK-NO-SME-ROUTINES-NEXT: cntd x9
; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp x9, x19, [sp, #80] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset w19, -8
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset vg, -16
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset w30, -24
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset w29, -32
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b8, -40
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b9, -48
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b10, -56
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b11, -64
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b12, -72
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b13, -80
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b14, -88
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_offset b15, -96
; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x1
; CHECK-NO-SME-ROUTINES-NEXT: mrs x19, SVCR
; CHECK-NO-SME-ROUTINES-NEXT: tbz w19, #0, .LBB4_2
; CHECK-NO-SME-ROUTINES-NEXT: // %bb.1: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: smstop sm
; CHECK-NO-SME-ROUTINES-NEXT: .LBB4_2: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: mov w1, #5 // =0x5
; CHECK-NO-SME-ROUTINES-NEXT: bl memchr
; CHECK-NO-SME-ROUTINES-NEXT: tbz w19, #0, .LBB4_4
; CHECK-NO-SME-ROUTINES-NEXT: // %bb.3: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: smstart sm
; CHECK-NO-SME-ROUTINES-NEXT: .LBB4_4: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldr x19, [sp, #88] // 8-byte Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_def_cfa_offset 0
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore w19
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore vg
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore w30
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore w29
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b8
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b9
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b10
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b11
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b12
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b13
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b14
; CHECK-NO-SME-ROUTINES-NEXT: .cfi_restore b15
; CHECK-NO-SME-ROUTINES-NEXT: ret
;
; CHECK-MOPS-LABEL: sc_memchr:
; CHECK-MOPS: // %bb.0: // %entry
; CHECK-MOPS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-MOPS-NEXT: .cfi_def_cfa_offset 16
; CHECK-MOPS-NEXT: .cfi_offset w30, -16
; CHECK-MOPS-NEXT: mov x2, x1
; CHECK-MOPS-NEXT: mov w1, #5 // =0x5
; CHECK-MOPS-NEXT: bl __arm_sc_memchr
; CHECK-MOPS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-MOPS-NEXT: ret
entry:
%res = tail call ptr @memchr(ptr %src, i32 5, i64 %n)
ret ptr %res
}
; Non-streaming[-compatible] call to memchr.
define ptr @ns_memcpy(ptr %src, i64 %n) {
; CHECK-COMMON-LABEL: ns_memcpy:
; CHECK-COMMON: // %bb.0: // %entry
; CHECK-COMMON-NEXT: mov x2, x1
; CHECK-COMMON-NEXT: mov w1, #5 // =0x5
; CHECK-COMMON-NEXT: b memchr
entry:
%res = tail call ptr @memchr(ptr %src, i32 5, i64 %n)
ret ptr %res
}
define void @sc_memcpy(i64 noundef %n) "aarch64_pstate_sm_compatible" nounwind {
; CHECK-LABEL: sc_memcpy:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: mov x2, x0
; CHECK-NEXT: adrp x0, :got:dst
; CHECK-NEXT: adrp x1, :got:src
; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NEXT: ldr x1, [x1, :got_lo12:src]
; CHECK-NEXT: bl __arm_sc_memcpy
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
;
; CHECK-NO-SME-ROUTINES-LABEL: sc_memcpy:
; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x0
; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst
; CHECK-NO-SME-ROUTINES-NEXT: adrp x1, :got:src
; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: mrs x19, SVCR
; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NO-SME-ROUTINES-NEXT: ldr x1, [x1, :got_lo12:src]
; CHECK-NO-SME-ROUTINES-NEXT: tbz w19, #0, .LBB6_2
; CHECK-NO-SME-ROUTINES-NEXT: // %bb.1: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: smstop sm
; CHECK-NO-SME-ROUTINES-NEXT: .LBB6_2: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: bl memcpy
; CHECK-NO-SME-ROUTINES-NEXT: tbz w19, #0, .LBB6_4
; CHECK-NO-SME-ROUTINES-NEXT: // %bb.3: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: smstart sm
; CHECK-NO-SME-ROUTINES-NEXT: .LBB6_4: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ret
;
; CHECK-MOPS-LABEL: sc_memcpy:
; CHECK-MOPS: // %bb.0: // %entry
; CHECK-MOPS-NEXT: adrp x8, :got:src
; CHECK-MOPS-NEXT: adrp x9, :got:dst
; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:src]
; CHECK-MOPS-NEXT: ldr x9, [x9, :got_lo12:dst]
; CHECK-MOPS-NEXT: cpyfp [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: cpyfm [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: cpyfe [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: ret
entry:
tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
ret void
}
define void @sb_memcpy(i64 noundef %n) "aarch64_pstate_sm_body" nounwind {
; CHECK-LABEL: sb_memcpy:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: mov x2, x0
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill
; CHECK-NEXT: smstart sm
; CHECK-NEXT: adrp x0, :got:dst
; CHECK-NEXT: adrp x1, :got:src
; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NEXT: ldr x1, [x1, :got_lo12:src]
; CHECK-NEXT: bl __arm_sc_memcpy
; CHECK-NEXT: smstop sm
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT: ret
;
; CHECK-NO-SME-ROUTINES-LABEL: sb_memcpy:
; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry
; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x0
; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NO-SME-ROUTINES-NEXT: str x30, [sp, #64] // 8-byte Spill
; CHECK-NO-SME-ROUTINES-NEXT: smstart sm
; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst
; CHECK-NO-SME-ROUTINES-NEXT: adrp x1, :got:src
; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]
; CHECK-NO-SME-ROUTINES-NEXT: ldr x1, [x1, :got_lo12:src]
; CHECK-NO-SME-ROUTINES-NEXT: smstop sm
; CHECK-NO-SME-ROUTINES-NEXT: bl memcpy
; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NO-SME-ROUTINES-NEXT: ret
;
; CHECK-MOPS-LABEL: sb_memcpy:
; CHECK-MOPS: // %bb.0: // %entry
; CHECK-MOPS-NEXT: stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
; CHECK-MOPS-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-MOPS-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-MOPS-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-MOPS-NEXT: smstart sm
; CHECK-MOPS-NEXT: adrp x8, :got:src
; CHECK-MOPS-NEXT: adrp x9, :got:dst
; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:src]
; CHECK-MOPS-NEXT: ldr x9, [x9, :got_lo12:dst]
; CHECK-MOPS-NEXT: cpyfp [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: cpyfm [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: cpyfe [x9]!, [x8]!, x0!
; CHECK-MOPS-NEXT: smstop sm
; CHECK-MOPS-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-MOPS-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-MOPS-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-MOPS-NEXT: ldp d15, d14, [sp], #64 // 16-byte Folded Reload
; CHECK-MOPS-NEXT: ret
entry:
tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
ret void
}
declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)
declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)
declare ptr @memchr(ptr, i32, i64)
Reference in New Issue View Git Blame Copy Permalink