shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll
Benjamin Maxwell d0c9599c41
[AArch64][SME] Use entry pstate.sm for conditional streaming-mode changes (#152169) 
We only do conditional streaming mode changes in two cases:

- Around calls in streaming-compatible functions that don't have a
streaming body
- At the entry/exit of streaming-compatible functions with a streaming
body

In both cases, the condition depends on the entry pstate.sm value. Given
this, we don't need to emit calls to __arm_sme_state at every mode
change.

This patch handles this by placing a "AArch64ISD::ENTRY_PSTATE_SM" node
in the entry block and copying the result to a register. The register is
then used whenever we need to emit a conditional streaming mode change.
The "ENTRY_PSTATE_SM" node expands to a call to "__arm_sme_state" only
if (after SelectionDAG) the function is determined to have
streaming-mode changes.

This has two main advantages:

1. It allows back-to-back conditional smstart/stop pairs to be folded
2. It has the correct behaviour for EH landing pads
- These are entered with pstate.sm = 0, and should switch mode based on
the entry pstate.sm
   - Note: This is not fully implemented yet
2025-08-12 09:15:30 +01:00

1209 lines
54 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s
; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -frame-pointer=non-leaf -verify-machineinstrs < %s | FileCheck %s --check-prefix=FP-CHECK
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -frame-pointer=non-leaf -verify-machineinstrs < %s | FileCheck %s --check-prefix=NO-SVE-CHECK
; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -verify-machineinstrs -enable-machine-outliner < %s | FileCheck %s --check-prefix=OUTLINER-CHECK
declare void @callee();
declare void @fixed_callee(<4 x i32>);
declare void @scalable_callee(<vscale x 2 x i64>);
declare void @streaming_callee() #0;
declare void @streaming_callee_with_arg(i32) #0;
; Simple example of a function with one call requiring a streaming mode change
;
define void @vg_unwind_simple() #0 {
; CHECK-LABEL: vg_unwind_simple:
; CHECK: // %bb.0:
; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 80
; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: stp x30, x9, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_offset w30, -16
; CHECK-NEXT: .cfi_offset b8, -24
; CHECK-NEXT: .cfi_offset b9, -32
; CHECK-NEXT: .cfi_offset b10, -40
; CHECK-NEXT: .cfi_offset b11, -48
; CHECK-NEXT: .cfi_offset b12, -56
; CHECK-NEXT: .cfi_offset b13, -64
; CHECK-NEXT: .cfi_offset b14, -72
; CHECK-NEXT: .cfi_offset b15, -80
; CHECK-NEXT: .cfi_offset vg, -8
; CHECK-NEXT: smstop sm
; CHECK-NEXT: bl callee
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Folded Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore b8
; CHECK-NEXT: .cfi_restore b9
; CHECK-NEXT: .cfi_restore b10
; CHECK-NEXT: .cfi_restore b11
; CHECK-NEXT: .cfi_restore b12
; CHECK-NEXT: .cfi_restore b13
; CHECK-NEXT: .cfi_restore b14
; CHECK-NEXT: .cfi_restore b15
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: vg_unwind_simple:
; FP-CHECK: // %bb.0:
; FP-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_def_cfa_offset 96
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; FP-CHECK-NEXT: str x9, [sp, #80] // 8-byte Folded Spill
; FP-CHECK-NEXT: add x29, sp, #64
; FP-CHECK-NEXT: .cfi_def_cfa w29, 32
; FP-CHECK-NEXT: .cfi_offset w30, -24
; FP-CHECK-NEXT: .cfi_offset w29, -32
; FP-CHECK-NEXT: .cfi_offset b8, -40
; FP-CHECK-NEXT: .cfi_offset b9, -48
; FP-CHECK-NEXT: .cfi_offset b10, -56
; FP-CHECK-NEXT: .cfi_offset b11, -64
; FP-CHECK-NEXT: .cfi_offset b12, -72
; FP-CHECK-NEXT: .cfi_offset b13, -80
; FP-CHECK-NEXT: .cfi_offset b14, -88
; FP-CHECK-NEXT: .cfi_offset b15, -96
; FP-CHECK-NEXT: .cfi_offset vg, -16
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: bl callee
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 96
; FP-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: .cfi_restore b8
; FP-CHECK-NEXT: .cfi_restore b9
; FP-CHECK-NEXT: .cfi_restore b10
; FP-CHECK-NEXT: .cfi_restore b11
; FP-CHECK-NEXT: .cfi_restore b12
; FP-CHECK-NEXT: .cfi_restore b13
; FP-CHECK-NEXT: .cfi_restore b14
; FP-CHECK-NEXT: .cfi_restore b15
; FP-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: vg_unwind_simple:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void @callee();
ret void;
}
; As above, with an extra register clobbered by the inline asm call which
; changes NeedsGapToAlignStack to false
;
define void @vg_unwind_needs_gap() #0 {
; CHECK-LABEL: vg_unwind_needs_gap:
; CHECK: // %bb.0:
; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: stp x30, x9, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: str x20, [sp, #80] // 8-byte Folded Spill
; CHECK-NEXT: .cfi_offset w20, -16
; CHECK-NEXT: .cfi_offset w30, -32
; CHECK-NEXT: .cfi_offset b8, -40
; CHECK-NEXT: .cfi_offset b9, -48
; CHECK-NEXT: .cfi_offset b10, -56
; CHECK-NEXT: .cfi_offset b11, -64
; CHECK-NEXT: .cfi_offset b12, -72
; CHECK-NEXT: .cfi_offset b13, -80
; CHECK-NEXT: .cfi_offset b14, -88
; CHECK-NEXT: .cfi_offset b15, -96
; CHECK-NEXT: //APP
; CHECK-NEXT: //NO_APP
; CHECK-NEXT: .cfi_offset vg, -24
; CHECK-NEXT: smstop sm
; CHECK-NEXT: bl callee
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldr x20, [sp, #80] // 8-byte Folded Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w20
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore b8
; CHECK-NEXT: .cfi_restore b9
; CHECK-NEXT: .cfi_restore b10
; CHECK-NEXT: .cfi_restore b11
; CHECK-NEXT: .cfi_restore b12
; CHECK-NEXT: .cfi_restore b13
; CHECK-NEXT: .cfi_restore b14
; CHECK-NEXT: .cfi_restore b15
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: vg_unwind_needs_gap:
; FP-CHECK: // %bb.0:
; FP-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_def_cfa_offset 96
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x9, x20, [sp, #80] // 16-byte Folded Spill
; FP-CHECK-NEXT: add x29, sp, #64
; FP-CHECK-NEXT: .cfi_def_cfa w29, 32
; FP-CHECK-NEXT: .cfi_offset w20, -8
; FP-CHECK-NEXT: .cfi_offset w30, -24
; FP-CHECK-NEXT: .cfi_offset w29, -32
; FP-CHECK-NEXT: .cfi_offset b8, -40
; FP-CHECK-NEXT: .cfi_offset b9, -48
; FP-CHECK-NEXT: .cfi_offset b10, -56
; FP-CHECK-NEXT: .cfi_offset b11, -64
; FP-CHECK-NEXT: .cfi_offset b12, -72
; FP-CHECK-NEXT: .cfi_offset b13, -80
; FP-CHECK-NEXT: .cfi_offset b14, -88
; FP-CHECK-NEXT: .cfi_offset b15, -96
; FP-CHECK-NEXT: //APP
; FP-CHECK-NEXT: //NO_APP
; FP-CHECK-NEXT: .cfi_offset vg, -16
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: bl callee
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 96
; FP-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldr x20, [sp, #88] // 8-byte Folded Reload
; FP-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w20
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: .cfi_restore b8
; FP-CHECK-NEXT: .cfi_restore b9
; FP-CHECK-NEXT: .cfi_restore b10
; FP-CHECK-NEXT: .cfi_restore b11
; FP-CHECK-NEXT: .cfi_restore b12
; FP-CHECK-NEXT: .cfi_restore b13
; FP-CHECK-NEXT: .cfi_restore b14
; FP-CHECK-NEXT: .cfi_restore b15
; FP-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: vg_unwind_needs_gap:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void asm sideeffect "", "~{x20}"()
call void @callee();
ret void;
}
define void @vg_unwind_with_fixed_args(<4 x i32> %x) #0 {
; CHECK-LABEL: vg_unwind_with_fixed_args:
; CHECK: // %bb.0:
; CHECK-NEXT: sub sp, sp, #96
; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: stp x30, x9, [sp, #80] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_offset w30, -16
; CHECK-NEXT: .cfi_offset b8, -24
; CHECK-NEXT: .cfi_offset b9, -32
; CHECK-NEXT: .cfi_offset b10, -40
; CHECK-NEXT: .cfi_offset b11, -48
; CHECK-NEXT: .cfi_offset b12, -56
; CHECK-NEXT: .cfi_offset b13, -64
; CHECK-NEXT: .cfi_offset b14, -72
; CHECK-NEXT: .cfi_offset b15, -80
; CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_offset vg, -8
; CHECK-NEXT: smstop sm
; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; CHECK-NEXT: bl fixed_callee
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT: ldr x30, [sp, #80] // 8-byte Folded Reload
; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: add sp, sp, #96
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore b8
; CHECK-NEXT: .cfi_restore b9
; CHECK-NEXT: .cfi_restore b10
; CHECK-NEXT: .cfi_restore b11
; CHECK-NEXT: .cfi_restore b12
; CHECK-NEXT: .cfi_restore b13
; CHECK-NEXT: .cfi_restore b14
; CHECK-NEXT: .cfi_restore b15
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: vg_unwind_with_fixed_args:
; FP-CHECK: // %bb.0:
; FP-CHECK-NEXT: sub sp, sp, #112
; FP-CHECK-NEXT: .cfi_def_cfa_offset 112
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x29, x30, [sp, #80] // 16-byte Folded Spill
; FP-CHECK-NEXT: str x9, [sp, #96] // 8-byte Folded Spill
; FP-CHECK-NEXT: add x29, sp, #80
; FP-CHECK-NEXT: .cfi_def_cfa w29, 32
; FP-CHECK-NEXT: .cfi_offset w30, -24
; FP-CHECK-NEXT: .cfi_offset w29, -32
; FP-CHECK-NEXT: .cfi_offset b8, -40
; FP-CHECK-NEXT: .cfi_offset b9, -48
; FP-CHECK-NEXT: .cfi_offset b10, -56
; FP-CHECK-NEXT: .cfi_offset b11, -64
; FP-CHECK-NEXT: .cfi_offset b12, -72
; FP-CHECK-NEXT: .cfi_offset b13, -80
; FP-CHECK-NEXT: .cfi_offset b14, -88
; FP-CHECK-NEXT: .cfi_offset b15, -96
; FP-CHECK-NEXT: str q0, [sp] // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_offset vg, -16
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload
; FP-CHECK-NEXT: bl fixed_callee
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 112
; FP-CHECK-NEXT: ldp x29, x30, [sp, #80] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload
; FP-CHECK-NEXT: add sp, sp, #112
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: .cfi_restore b8
; FP-CHECK-NEXT: .cfi_restore b9
; FP-CHECK-NEXT: .cfi_restore b10
; FP-CHECK-NEXT: .cfi_restore b11
; FP-CHECK-NEXT: .cfi_restore b12
; FP-CHECK-NEXT: .cfi_restore b13
; FP-CHECK-NEXT: .cfi_restore b14
; FP-CHECK-NEXT: .cfi_restore b15
; FP-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: vg_unwind_with_fixed_args:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void @fixed_callee(<4 x i32> %x);
ret void;
}
define void @vg_unwind_with_sve_args(<vscale x 2 x i64> %x) #0 {
; CHECK-LABEL: vg_unwind_with_sve_args:
; CHECK: // %bb.0:
; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp x9, x28, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_offset w28, -8
; CHECK-NEXT: .cfi_offset w30, -24
; CHECK-NEXT: .cfi_offset w29, -32
; CHECK-NEXT: addvl sp, sp, #-18
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 32 + 144 * VG
; CHECK-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
; CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
; CHECK-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
; CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
; CHECK-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
; CHECK-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
; CHECK-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
; CHECK-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
; CHECK-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d8 @ cfa - 8 * VG - 32
; CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d9 @ cfa - 16 * VG - 32
; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d10 @ cfa - 24 * VG - 32
; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d11 @ cfa - 32 * VG - 32
; CHECK-NEXT: .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d12 @ cfa - 40 * VG - 32
; CHECK-NEXT: .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d13 @ cfa - 48 * VG - 32
; CHECK-NEXT: .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d14 @ cfa - 56 * VG - 32
; CHECK-NEXT: .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d15 @ cfa - 64 * VG - 32
; CHECK-NEXT: addvl sp, sp, #-1
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x01, 0x1e, 0x22 // sp + 32 + 152 * VG
; CHECK-NEXT: str z0, [sp] // 16-byte Folded Spill
; CHECK-NEXT: //APP
; CHECK-NEXT: //NO_APP
; CHECK-NEXT: .cfi_offset vg, -16
; CHECK-NEXT: smstop sm
; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload
; CHECK-NEXT: bl scalable_callee
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: addvl sp, sp, #1
; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 32 + 144 * VG
; CHECK-NEXT: ptrue pn8.b
; CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
; CHECK-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
; CHECK-NEXT: addvl sp, sp, #18
; CHECK-NEXT: .cfi_def_cfa wsp, 32
; CHECK-NEXT: .cfi_restore z8
; CHECK-NEXT: .cfi_restore z9
; CHECK-NEXT: .cfi_restore z10
; CHECK-NEXT: .cfi_restore z11
; CHECK-NEXT: .cfi_restore z12
; CHECK-NEXT: .cfi_restore z13
; CHECK-NEXT: .cfi_restore z14
; CHECK-NEXT: .cfi_restore z15
; CHECK-NEXT: ldr x28, [sp, #24] // 8-byte Folded Reload
; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w28
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: vg_unwind_with_sve_args:
; FP-CHECK: // %bb.0:
; FP-CHECK-NEXT: stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_def_cfa_offset 48
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: str x9, [sp, #16] // 8-byte Folded Spill
; FP-CHECK-NEXT: mov x29, sp
; FP-CHECK-NEXT: .cfi_def_cfa w29, 48
; FP-CHECK-NEXT: .cfi_offset w27, -8
; FP-CHECK-NEXT: .cfi_offset w28, -16
; FP-CHECK-NEXT: .cfi_offset w30, -40
; FP-CHECK-NEXT: .cfi_offset w29, -48
; FP-CHECK-NEXT: addvl sp, sp, #-18
; FP-CHECK-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: ptrue pn8.b
; FP-CHECK-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: st1b { z22.b, z23.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill
; FP-CHECK-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
; FP-CHECK-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #6, mul vl] // 32-byte Folded Spill
; FP-CHECK-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
; FP-CHECK-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #10, mul vl] // 32-byte Folded Spill
; FP-CHECK-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
; FP-CHECK-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #14, mul vl] // 32-byte Folded Spill
; FP-CHECK-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
; FP-CHECK-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
; FP-CHECK-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d8 @ cfa - 8 * VG - 48
; FP-CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d9 @ cfa - 16 * VG - 48
; FP-CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d10 @ cfa - 24 * VG - 48
; FP-CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d11 @ cfa - 32 * VG - 48
; FP-CHECK-NEXT: .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d12 @ cfa - 40 * VG - 48
; FP-CHECK-NEXT: .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d13 @ cfa - 48 * VG - 48
; FP-CHECK-NEXT: .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d14 @ cfa - 56 * VG - 48
; FP-CHECK-NEXT: .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d15 @ cfa - 64 * VG - 48
; FP-CHECK-NEXT: addvl sp, sp, #-1
; FP-CHECK-NEXT: str z0, [x29, #-19, mul vl] // 16-byte Folded Spill
; FP-CHECK-NEXT: //APP
; FP-CHECK-NEXT: //NO_APP
; FP-CHECK-NEXT: .cfi_offset vg, -32
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: ldr z0, [x29, #-19, mul vl] // 16-byte Folded Reload
; FP-CHECK-NEXT: bl scalable_callee
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: addvl sp, sp, #1
; FP-CHECK-NEXT: ptrue pn8.b
; FP-CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
; FP-CHECK-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload
; FP-CHECK-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
; FP-CHECK-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #6, mul vl] // 32-byte Folded Reload
; FP-CHECK-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
; FP-CHECK-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #10, mul vl] // 32-byte Folded Reload
; FP-CHECK-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
; FP-CHECK-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #14, mul vl] // 32-byte Folded Reload
; FP-CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
; FP-CHECK-NEXT: addvl sp, sp, #18
; FP-CHECK-NEXT: .cfi_restore z8
; FP-CHECK-NEXT: .cfi_restore z9
; FP-CHECK-NEXT: .cfi_restore z10
; FP-CHECK-NEXT: .cfi_restore z11
; FP-CHECK-NEXT: .cfi_restore z12
; FP-CHECK-NEXT: .cfi_restore z13
; FP-CHECK-NEXT: .cfi_restore z14
; FP-CHECK-NEXT: .cfi_restore z15
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 48
; FP-CHECK-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp x29, x30, [sp], #48 // 16-byte Folded Reload
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w27
; FP-CHECK-NEXT: .cfi_restore w28
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: vg_unwind_with_sve_args:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void asm sideeffect "", "~{x28}"()
call void @scalable_callee(<vscale x 2 x i64> %x);
ret void;
}
; This test was based on stack-probing-64k.ll and tries to test multiple uses of
; findScratchNonCalleeSaveRegister.
;
define void @vg_unwind_multiple_scratch_regs(ptr %out) #1 {
; CHECK-LABEL: vg_unwind_multiple_scratch_regs:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: str x9, [sp, #80] // 8-byte Folded Spill
; CHECK-NEXT: .cfi_offset w30, -24
; CHECK-NEXT: .cfi_offset w29, -32
; CHECK-NEXT: .cfi_offset b8, -40
; CHECK-NEXT: .cfi_offset b9, -48
; CHECK-NEXT: .cfi_offset b10, -56
; CHECK-NEXT: .cfi_offset b11, -64
; CHECK-NEXT: .cfi_offset b12, -72
; CHECK-NEXT: .cfi_offset b13, -80
; CHECK-NEXT: .cfi_offset b14, -88
; CHECK-NEXT: .cfi_offset b15, -96
; CHECK-NEXT: sub x9, sp, #80, lsl #12 // =327680
; CHECK-NEXT: .cfi_def_cfa w9, 327776
; CHECK-NEXT: .LBB4_1: // %entry
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; CHECK-NEXT: cmp sp, x9
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b.ne .LBB4_1
; CHECK-NEXT: // %bb.2: // %entry
; CHECK-NEXT: .cfi_def_cfa_register wsp
; CHECK-NEXT: mov x8, sp
; CHECK-NEXT: str x8, [x0]
; CHECK-NEXT: .cfi_offset vg, -16
; CHECK-NEXT: smstop sm
; CHECK-NEXT: bl callee
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: add sp, sp, #80, lsl #12 // =327680
; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: .cfi_restore b8
; CHECK-NEXT: .cfi_restore b9
; CHECK-NEXT: .cfi_restore b10
; CHECK-NEXT: .cfi_restore b11
; CHECK-NEXT: .cfi_restore b12
; CHECK-NEXT: .cfi_restore b13
; CHECK-NEXT: .cfi_restore b14
; CHECK-NEXT: .cfi_restore b15
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: vg_unwind_multiple_scratch_regs:
; FP-CHECK: // %bb.0: // %entry
; FP-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_def_cfa_offset 96
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x9, x28, [sp, #80] // 16-byte Folded Spill
; FP-CHECK-NEXT: add x29, sp, #64
; FP-CHECK-NEXT: .cfi_def_cfa w29, 32
; FP-CHECK-NEXT: .cfi_offset w28, -8
; FP-CHECK-NEXT: .cfi_offset w30, -24
; FP-CHECK-NEXT: .cfi_offset w29, -32
; FP-CHECK-NEXT: .cfi_offset b8, -40
; FP-CHECK-NEXT: .cfi_offset b9, -48
; FP-CHECK-NEXT: .cfi_offset b10, -56
; FP-CHECK-NEXT: .cfi_offset b11, -64
; FP-CHECK-NEXT: .cfi_offset b12, -72
; FP-CHECK-NEXT: .cfi_offset b13, -80
; FP-CHECK-NEXT: .cfi_offset b14, -88
; FP-CHECK-NEXT: .cfi_offset b15, -96
; FP-CHECK-NEXT: sub x9, sp, #80, lsl #12 // =327680
; FP-CHECK-NEXT: .LBB4_1: // %entry
; FP-CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; FP-CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; FP-CHECK-NEXT: cmp sp, x9
; FP-CHECK-NEXT: str xzr, [sp]
; FP-CHECK-NEXT: b.ne .LBB4_1
; FP-CHECK-NEXT: // %bb.2: // %entry
; FP-CHECK-NEXT: mov x8, sp
; FP-CHECK-NEXT: str x8, [x0]
; FP-CHECK-NEXT: .cfi_offset vg, -16
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: bl callee
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: add sp, sp, #80, lsl #12 // =327680
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 96
; FP-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldr x28, [sp, #88] // 8-byte Folded Reload
; FP-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w28
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: .cfi_restore b8
; FP-CHECK-NEXT: .cfi_restore b9
; FP-CHECK-NEXT: .cfi_restore b10
; FP-CHECK-NEXT: .cfi_restore b11
; FP-CHECK-NEXT: .cfi_restore b12
; FP-CHECK-NEXT: .cfi_restore b13
; FP-CHECK-NEXT: .cfi_restore b14
; FP-CHECK-NEXT: .cfi_restore b15
; FP-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: vg_unwind_multiple_scratch_regs:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
entry:
%v = alloca i8, i64 327680, align 1
store ptr %v, ptr %out, align 8
call void @callee()
ret void
}
; Locally streaming functions require storing both the streaming and
; non-streaming values of VG.
;
define void @vg_locally_streaming_fn() #3 {
; CHECK-LABEL: vg_locally_streaming_fn:
; CHECK: // %bb.0:
; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: rdsvl x9, #1
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: lsr x9, x9, #3
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: stp x30, x9, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: cntd x9
; CHECK-NEXT: str x9, [sp, #80] // 8-byte Folded Spill
; CHECK-NEXT: .cfi_offset vg, -16
; CHECK-NEXT: .cfi_offset w30, -32
; CHECK-NEXT: .cfi_offset b8, -40
; CHECK-NEXT: .cfi_offset b9, -48
; CHECK-NEXT: .cfi_offset b10, -56
; CHECK-NEXT: .cfi_offset b11, -64
; CHECK-NEXT: .cfi_offset b12, -72
; CHECK-NEXT: .cfi_offset b13, -80
; CHECK-NEXT: .cfi_offset b14, -88
; CHECK-NEXT: .cfi_offset b15, -96
; CHECK-NEXT: bl callee
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: bl streaming_callee
; CHECK-NEXT: .cfi_offset vg, -24
; CHECK-NEXT: smstop sm
; CHECK-NEXT: bl callee
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Folded Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore b8
; CHECK-NEXT: .cfi_restore b9
; CHECK-NEXT: .cfi_restore b10
; CHECK-NEXT: .cfi_restore b11
; CHECK-NEXT: .cfi_restore b12
; CHECK-NEXT: .cfi_restore b13
; CHECK-NEXT: .cfi_restore b14
; CHECK-NEXT: .cfi_restore b15
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: vg_locally_streaming_fn:
; FP-CHECK: // %bb.0:
; FP-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_def_cfa_offset 96
; FP-CHECK-NEXT: rdsvl x9, #1
; FP-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; FP-CHECK-NEXT: lsr x9, x9, #3
; FP-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; FP-CHECK-NEXT: str x9, [sp, #80] // 8-byte Folded Spill
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; FP-CHECK-NEXT: str x9, [sp, #88] // 8-byte Folded Spill
; FP-CHECK-NEXT: add x29, sp, #64
; FP-CHECK-NEXT: .cfi_def_cfa w29, 32
; FP-CHECK-NEXT: .cfi_offset vg, -8
; FP-CHECK-NEXT: .cfi_offset w30, -24
; FP-CHECK-NEXT: .cfi_offset w29, -32
; FP-CHECK-NEXT: .cfi_offset b8, -40
; FP-CHECK-NEXT: .cfi_offset b9, -48
; FP-CHECK-NEXT: .cfi_offset b10, -56
; FP-CHECK-NEXT: .cfi_offset b11, -64
; FP-CHECK-NEXT: .cfi_offset b12, -72
; FP-CHECK-NEXT: .cfi_offset b13, -80
; FP-CHECK-NEXT: .cfi_offset b14, -88
; FP-CHECK-NEXT: .cfi_offset b15, -96
; FP-CHECK-NEXT: bl callee
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: bl streaming_callee
; FP-CHECK-NEXT: .cfi_offset vg, -16
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: bl callee
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 96
; FP-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: .cfi_restore b8
; FP-CHECK-NEXT: .cfi_restore b9
; FP-CHECK-NEXT: .cfi_restore b10
; FP-CHECK-NEXT: .cfi_restore b11
; FP-CHECK-NEXT: .cfi_restore b12
; FP-CHECK-NEXT: .cfi_restore b13
; FP-CHECK-NEXT: .cfi_restore b14
; FP-CHECK-NEXT: .cfi_restore b15
; FP-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: vg_locally_streaming_fn:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void @callee()
call void @streaming_callee()
call void @callee()
ret void
}
define void @streaming_compatible_to_streaming() #4 {
; CHECK-LABEL: streaming_compatible_to_streaming:
; CHECK: // %bb.0:
; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: stp x30, x9, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: str x19, [sp, #80] // 8-byte Folded Spill
; CHECK-NEXT: .cfi_offset w19, -16
; CHECK-NEXT: .cfi_offset w30, -32
; CHECK-NEXT: .cfi_offset b8, -40
; CHECK-NEXT: .cfi_offset b9, -48
; CHECK-NEXT: .cfi_offset b10, -56
; CHECK-NEXT: .cfi_offset b11, -64
; CHECK-NEXT: .cfi_offset b12, -72
; CHECK-NEXT: .cfi_offset b13, -80
; CHECK-NEXT: .cfi_offset b14, -88
; CHECK-NEXT: .cfi_offset b15, -96
; CHECK-NEXT: bl __arm_sme_state
; CHECK-NEXT: mov x19, x0
; CHECK-NEXT: .cfi_offset vg, -24
; CHECK-NEXT: tbnz w19, #0, .LBB6_2
; CHECK-NEXT: // %bb.1:
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .LBB6_2:
; CHECK-NEXT: bl streaming_callee
; CHECK-NEXT: tbnz w19, #0, .LBB6_4
; CHECK-NEXT: // %bb.3:
; CHECK-NEXT: smstop sm
; CHECK-NEXT: .LBB6_4:
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldr x19, [sp, #80] // 8-byte Folded Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w19
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore b8
; CHECK-NEXT: .cfi_restore b9
; CHECK-NEXT: .cfi_restore b10
; CHECK-NEXT: .cfi_restore b11
; CHECK-NEXT: .cfi_restore b12
; CHECK-NEXT: .cfi_restore b13
; CHECK-NEXT: .cfi_restore b14
; CHECK-NEXT: .cfi_restore b15
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: streaming_compatible_to_streaming:
; FP-CHECK: // %bb.0:
; FP-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_def_cfa_offset 96
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x9, x19, [sp, #80] // 16-byte Folded Spill
; FP-CHECK-NEXT: add x29, sp, #64
; FP-CHECK-NEXT: .cfi_def_cfa w29, 32
; FP-CHECK-NEXT: .cfi_offset w19, -8
; FP-CHECK-NEXT: .cfi_offset w30, -24
; FP-CHECK-NEXT: .cfi_offset w29, -32
; FP-CHECK-NEXT: .cfi_offset b8, -40
; FP-CHECK-NEXT: .cfi_offset b9, -48
; FP-CHECK-NEXT: .cfi_offset b10, -56
; FP-CHECK-NEXT: .cfi_offset b11, -64
; FP-CHECK-NEXT: .cfi_offset b12, -72
; FP-CHECK-NEXT: .cfi_offset b13, -80
; FP-CHECK-NEXT: .cfi_offset b14, -88
; FP-CHECK-NEXT: .cfi_offset b15, -96
; FP-CHECK-NEXT: bl __arm_sme_state
; FP-CHECK-NEXT: mov x19, x0
; FP-CHECK-NEXT: .cfi_offset vg, -16
; FP-CHECK-NEXT: tbnz w19, #0, .LBB6_2
; FP-CHECK-NEXT: // %bb.1:
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .LBB6_2:
; FP-CHECK-NEXT: bl streaming_callee
; FP-CHECK-NEXT: tbnz w19, #0, .LBB6_4
; FP-CHECK-NEXT: // %bb.3:
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: .LBB6_4:
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 96
; FP-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldr x19, [sp, #88] // 8-byte Folded Reload
; FP-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w19
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: .cfi_restore b8
; FP-CHECK-NEXT: .cfi_restore b9
; FP-CHECK-NEXT: .cfi_restore b10
; FP-CHECK-NEXT: .cfi_restore b11
; FP-CHECK-NEXT: .cfi_restore b12
; FP-CHECK-NEXT: .cfi_restore b13
; FP-CHECK-NEXT: .cfi_restore b14
; FP-CHECK-NEXT: .cfi_restore b15
; FP-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: streaming_compatible_to_streaming:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void @streaming_callee()
ret void
}
define void @streaming_compatible_to_non_streaming() #4 {
; CHECK-LABEL: streaming_compatible_to_non_streaming:
; CHECK: // %bb.0:
; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 96
; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: stp x30, x9, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: str x19, [sp, #80] // 8-byte Folded Spill
; CHECK-NEXT: .cfi_offset w19, -16
; CHECK-NEXT: .cfi_offset w30, -32
; CHECK-NEXT: .cfi_offset b8, -40
; CHECK-NEXT: .cfi_offset b9, -48
; CHECK-NEXT: .cfi_offset b10, -56
; CHECK-NEXT: .cfi_offset b11, -64
; CHECK-NEXT: .cfi_offset b12, -72
; CHECK-NEXT: .cfi_offset b13, -80
; CHECK-NEXT: .cfi_offset b14, -88
; CHECK-NEXT: .cfi_offset b15, -96
; CHECK-NEXT: bl __arm_sme_state
; CHECK-NEXT: mov x19, x0
; CHECK-NEXT: .cfi_offset vg, -24
; CHECK-NEXT: tbz w19, #0, .LBB7_2
; CHECK-NEXT: // %bb.1:
; CHECK-NEXT: smstop sm
; CHECK-NEXT: .LBB7_2:
; CHECK-NEXT: bl callee
; CHECK-NEXT: tbz w19, #0, .LBB7_4
; CHECK-NEXT: // %bb.3:
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .LBB7_4:
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldr x19, [sp, #80] // 8-byte Folded Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w19
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore b8
; CHECK-NEXT: .cfi_restore b9
; CHECK-NEXT: .cfi_restore b10
; CHECK-NEXT: .cfi_restore b11
; CHECK-NEXT: .cfi_restore b12
; CHECK-NEXT: .cfi_restore b13
; CHECK-NEXT: .cfi_restore b14
; CHECK-NEXT: .cfi_restore b15
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: streaming_compatible_to_non_streaming:
; FP-CHECK: // %bb.0:
; FP-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_def_cfa_offset 96
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x9, x19, [sp, #80] // 16-byte Folded Spill
; FP-CHECK-NEXT: add x29, sp, #64
; FP-CHECK-NEXT: .cfi_def_cfa w29, 32
; FP-CHECK-NEXT: .cfi_offset w19, -8
; FP-CHECK-NEXT: .cfi_offset w30, -24
; FP-CHECK-NEXT: .cfi_offset w29, -32
; FP-CHECK-NEXT: .cfi_offset b8, -40
; FP-CHECK-NEXT: .cfi_offset b9, -48
; FP-CHECK-NEXT: .cfi_offset b10, -56
; FP-CHECK-NEXT: .cfi_offset b11, -64
; FP-CHECK-NEXT: .cfi_offset b12, -72
; FP-CHECK-NEXT: .cfi_offset b13, -80
; FP-CHECK-NEXT: .cfi_offset b14, -88
; FP-CHECK-NEXT: .cfi_offset b15, -96
; FP-CHECK-NEXT: bl __arm_sme_state
; FP-CHECK-NEXT: mov x19, x0
; FP-CHECK-NEXT: .cfi_offset vg, -16
; FP-CHECK-NEXT: tbz w19, #0, .LBB7_2
; FP-CHECK-NEXT: // %bb.1:
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: .LBB7_2:
; FP-CHECK-NEXT: bl callee
; FP-CHECK-NEXT: tbz w19, #0, .LBB7_4
; FP-CHECK-NEXT: // %bb.3:
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .LBB7_4:
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 96
; FP-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldr x19, [sp, #88] // 8-byte Folded Reload
; FP-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w19
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: .cfi_restore b8
; FP-CHECK-NEXT: .cfi_restore b9
; FP-CHECK-NEXT: .cfi_restore b10
; FP-CHECK-NEXT: .cfi_restore b11
; FP-CHECK-NEXT: .cfi_restore b12
; FP-CHECK-NEXT: .cfi_restore b13
; FP-CHECK-NEXT: .cfi_restore b14
; FP-CHECK-NEXT: .cfi_restore b15
; FP-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: streaming_compatible_to_non_streaming:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void @callee()
ret void
}
; If the target does not have SVE, do not emit cntd in the prologue and
; instead spill the result returned by __arm_get_current_vg.
; This requires preserving the argument %x as the vg value is returned
; in X0.
;
define void @streaming_compatible_no_sve(i32 noundef %x) #4 {
; NO-SVE-CHECK-LABEL: streaming_compatible_no_sve:
; NO-SVE-CHECK: // %bb.0:
; NO-SVE-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: .cfi_def_cfa_offset 96
; NO-SVE-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: mov x9, x0
; NO-SVE-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: bl __arm_get_current_vg
; NO-SVE-CHECK-NEXT: stp x0, x19, [sp, #80] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: mov x0, x9
; NO-SVE-CHECK-NEXT: add x29, sp, #64
; NO-SVE-CHECK-NEXT: .cfi_def_cfa w29, 32
; NO-SVE-CHECK-NEXT: .cfi_offset w19, -8
; NO-SVE-CHECK-NEXT: .cfi_offset w30, -24
; NO-SVE-CHECK-NEXT: .cfi_offset w29, -32
; NO-SVE-CHECK-NEXT: .cfi_offset b8, -40
; NO-SVE-CHECK-NEXT: .cfi_offset b9, -48
; NO-SVE-CHECK-NEXT: .cfi_offset b10, -56
; NO-SVE-CHECK-NEXT: .cfi_offset b11, -64
; NO-SVE-CHECK-NEXT: .cfi_offset b12, -72
; NO-SVE-CHECK-NEXT: .cfi_offset b13, -80
; NO-SVE-CHECK-NEXT: .cfi_offset b14, -88
; NO-SVE-CHECK-NEXT: .cfi_offset b15, -96
; NO-SVE-CHECK-NEXT: mov w8, w0
; NO-SVE-CHECK-NEXT: bl __arm_sme_state
; NO-SVE-CHECK-NEXT: mov x19, x0
; NO-SVE-CHECK-NEXT: .cfi_offset vg, -16
; NO-SVE-CHECK-NEXT: tbnz w19, #0, .LBB8_2
; NO-SVE-CHECK-NEXT: // %bb.1:
; NO-SVE-CHECK-NEXT: smstart sm
; NO-SVE-CHECK-NEXT: .LBB8_2:
; NO-SVE-CHECK-NEXT: mov w0, w8
; NO-SVE-CHECK-NEXT: bl streaming_callee_with_arg
; NO-SVE-CHECK-NEXT: tbnz w19, #0, .LBB8_4
; NO-SVE-CHECK-NEXT: // %bb.3:
; NO-SVE-CHECK-NEXT: smstop sm
; NO-SVE-CHECK-NEXT: .LBB8_4:
; NO-SVE-CHECK-NEXT: .cfi_restore vg
; NO-SVE-CHECK-NEXT: .cfi_def_cfa wsp, 96
; NO-SVE-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldr x19, [sp, #88] // 8-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: .cfi_def_cfa_offset 0
; NO-SVE-CHECK-NEXT: .cfi_restore w19
; NO-SVE-CHECK-NEXT: .cfi_restore w30
; NO-SVE-CHECK-NEXT: .cfi_restore w29
; NO-SVE-CHECK-NEXT: .cfi_restore b8
; NO-SVE-CHECK-NEXT: .cfi_restore b9
; NO-SVE-CHECK-NEXT: .cfi_restore b10
; NO-SVE-CHECK-NEXT: .cfi_restore b11
; NO-SVE-CHECK-NEXT: .cfi_restore b12
; NO-SVE-CHECK-NEXT: .cfi_restore b13
; NO-SVE-CHECK-NEXT: .cfi_restore b14
; NO-SVE-CHECK-NEXT: .cfi_restore b15
; NO-SVE-CHECK-NEXT: ret
;
; OUTLINER-CHECK-LABEL: streaming_compatible_no_sve:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void @streaming_callee_with_arg(i32 %x)
ret void
}
; The algorithm that fixes up the offsets of the callee-save/restore
; instructions must jump over the instructions that instantiate the current
; 'VG' value. We must make sure that it doesn't consider any RDSVL in
; user-code as if it is part of the frame-setup when doing so.
define void @test_rdsvl_right_after_prologue(i64 %x0) nounwind {
; NO-SVE-CHECK-LABEL: test_rdsvl_right_after_prologue:
; NO-SVE-CHECK: // %bb.0:
; NO-SVE-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: mov x9, x0
; NO-SVE-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; NO-SVE-CHECK-NEXT: bl __arm_get_current_vg
; NO-SVE-CHECK-NEXT: str x0, [sp, #80] // 8-byte Folded Spill
; NO-SVE-CHECK-NEXT: mov x0, x9
; NO-SVE-CHECK-NEXT: rdsvl x8, #1
; NO-SVE-CHECK-NEXT: add x29, sp, #64
; NO-SVE-CHECK-NEXT: lsr x8, x8, #3
; NO-SVE-CHECK-NEXT: mov x1, x0
; NO-SVE-CHECK-NEXT: smstart sm
; NO-SVE-CHECK-NEXT: mov x0, x8
; NO-SVE-CHECK-NEXT: bl bar
; NO-SVE-CHECK-NEXT: smstop sm
; NO-SVE-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; NO-SVE-CHECK-NEXT: ret
%some_alloc = alloca i64, align 8
%rdsvl = tail call i64 @llvm.aarch64.sme.cntsd()
call void @bar(i64 %rdsvl, i64 %x0)
ret void
}
declare void @bar(i64, i64) "aarch64_pstate_sm_enabled"
; Ensure we still emit async unwind information with -fno-asynchronous-unwind-tables
; if the function contains a streaming-mode change.
define void @vg_unwind_noasync() #5 {
; CHECK-LABEL: vg_unwind_noasync:
; CHECK: // %bb.0:
; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 80
; CHECK-NEXT: cntd x9
; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: stp x30, x9, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT: .cfi_offset w30, -16
; CHECK-NEXT: .cfi_offset b8, -24
; CHECK-NEXT: .cfi_offset b9, -32
; CHECK-NEXT: .cfi_offset b10, -40
; CHECK-NEXT: .cfi_offset b11, -48
; CHECK-NEXT: .cfi_offset b12, -56
; CHECK-NEXT: .cfi_offset b13, -64
; CHECK-NEXT: .cfi_offset b14, -72
; CHECK-NEXT: .cfi_offset b15, -80
; CHECK-NEXT: .cfi_offset vg, -8
; CHECK-NEXT: smstop sm
; CHECK-NEXT: bl callee
; CHECK-NEXT: smstart sm
; CHECK-NEXT: .cfi_restore vg
; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Folded Reload
; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore b8
; CHECK-NEXT: .cfi_restore b9
; CHECK-NEXT: .cfi_restore b10
; CHECK-NEXT: .cfi_restore b11
; CHECK-NEXT: .cfi_restore b12
; CHECK-NEXT: .cfi_restore b13
; CHECK-NEXT: .cfi_restore b14
; CHECK-NEXT: .cfi_restore b15
; CHECK-NEXT: ret
;
; FP-CHECK-LABEL: vg_unwind_noasync:
; FP-CHECK: // %bb.0:
; FP-CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; FP-CHECK-NEXT: .cfi_def_cfa_offset 96
; FP-CHECK-NEXT: cntd x9
; FP-CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill
; FP-CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill
; FP-CHECK-NEXT: str x9, [sp, #80] // 8-byte Folded Spill
; FP-CHECK-NEXT: add x29, sp, #64
; FP-CHECK-NEXT: .cfi_def_cfa w29, 32
; FP-CHECK-NEXT: .cfi_offset w30, -24
; FP-CHECK-NEXT: .cfi_offset w29, -32
; FP-CHECK-NEXT: .cfi_offset b8, -40
; FP-CHECK-NEXT: .cfi_offset b9, -48
; FP-CHECK-NEXT: .cfi_offset b10, -56
; FP-CHECK-NEXT: .cfi_offset b11, -64
; FP-CHECK-NEXT: .cfi_offset b12, -72
; FP-CHECK-NEXT: .cfi_offset b13, -80
; FP-CHECK-NEXT: .cfi_offset b14, -88
; FP-CHECK-NEXT: .cfi_offset b15, -96
; FP-CHECK-NEXT: .cfi_offset vg, -16
; FP-CHECK-NEXT: smstop sm
; FP-CHECK-NEXT: bl callee
; FP-CHECK-NEXT: smstart sm
; FP-CHECK-NEXT: .cfi_restore vg
; FP-CHECK-NEXT: .cfi_def_cfa wsp, 96
; FP-CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; FP-CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; FP-CHECK-NEXT: .cfi_def_cfa_offset 0
; FP-CHECK-NEXT: .cfi_restore w30
; FP-CHECK-NEXT: .cfi_restore w29
; FP-CHECK-NEXT: .cfi_restore b8
; FP-CHECK-NEXT: .cfi_restore b9
; FP-CHECK-NEXT: .cfi_restore b10
; FP-CHECK-NEXT: .cfi_restore b11
; FP-CHECK-NEXT: .cfi_restore b12
; FP-CHECK-NEXT: .cfi_restore b13
; FP-CHECK-NEXT: .cfi_restore b14
; FP-CHECK-NEXT: .cfi_restore b15
; FP-CHECK-NEXT: ret
; OUTLINER-CHECK-LABEL: vg_unwind_noasync:
; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
;
call void @callee();
ret void;
}
attributes #0 = { "aarch64_pstate_sm_enabled" uwtable(async) }
attributes #1 = { "probe-stack"="inline-asm" "aarch64_pstate_sm_enabled" uwtable(async) }
attributes #3 = { "aarch64_pstate_sm_body" uwtable(async) }
attributes #4 = { "aarch64_pstate_sm_compatible" uwtable(async) }
attributes #5 = { "aarch64_pstate_sm_enabled" }
Reference in New Issue View Git Blame Copy Permalink