eb764040bc
## Short Summary
This patch adds a new pass `aarch64-machine-sme-abi` to handle the ABI
for ZA state (e.g., lazy saves and agnostic ZA functions). This is
currently not enabled by default (but aims to be by LLVM 22). The goal
is for this new pass to more optimally place ZA saves/restores and to
work with exception handling.
## Long Description
This patch reimplements management of ZA state for functions with
private and shared ZA state. Agnostic ZA functions will be handled in a
later patch. For now, this is under the flag `-aarch64-new-sme-abi`,
however, we intend for this to replace the current SelectionDAG
implementation once complete.
The approach taken here is to mark instructions as needing ZA to be in a
specific ("ACTIVE" or "LOCAL_SAVED"). Machine instructions implicitly
defining or using ZA registers (such as $zt0 or $zab0) require the
"ACTIVE" state. Function calls may need the "LOCAL_SAVED" or "ACTIVE"
state depending on the callee (having shared or private ZA).
We already add ZA register uses/definitions to machine instructions, so
no extra work is needed to mark these.
Calls need to be marked by glueing Arch64ISD::INOUT_ZA_USE or
Arch64ISD::REQUIRES_ZA_SAVE to the CALLSEQ_START.
These markers are then used by the MachineSMEABIPass to find
instructions where there is a transition between required ZA states.
These are the points we need to insert code to set up or restore a ZA
save (or initialize ZA).
To handle control flow between blocks (which may have different ZA state
requirements), we bundle the incoming and outgoing edges of blocks.
Bundles are formed by assigning each block an incoming and outgoing
bundle (initially, all blocks have their own two bundles). Bundles are
then combined by joining the outgoing bundle of a block with the
incoming bundle of all successors.
These bundles are then assigned a ZA state based on the blocks that
participate in the bundle. Blocks whose incoming edges are in a bundle
"vote" for a ZA state that matches the state required at the first
instruction in the block, and likewise, blocks whose outgoing edges are
in a bundle vote for the ZA state that matches the last instruction in
the block. The ZA state with the most votes is used, which aims to
minimize the number of state transitions.
122 lines
4.8 KiB
LLVM
122 lines
4.8 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs < %s | FileCheck %s
|
|
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs -aarch64-new-sme-abi < %s | FileCheck %s --check-prefix=CHECK-NEWLOWERING
|
|
|
|
declare void @private_za_callee()
|
|
|
|
; Ensure that we don't use tail call optimization when a lazy-save is required.
|
|
define void @disable_tailcallopt() "aarch64_inout_za" nounwind {
|
|
; CHECK-LABEL: disable_tailcallopt:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
|
|
; CHECK-NEXT: mov x29, sp
|
|
; CHECK-NEXT: sub sp, sp, #16
|
|
; CHECK-NEXT: rdsvl x8, #1
|
|
; CHECK-NEXT: mov x9, sp
|
|
; CHECK-NEXT: msub x9, x8, x8, x9
|
|
; CHECK-NEXT: mov sp, x9
|
|
; CHECK-NEXT: stur x9, [x29, #-16]
|
|
; CHECK-NEXT: sub x9, x29, #16
|
|
; CHECK-NEXT: sturh wzr, [x29, #-6]
|
|
; CHECK-NEXT: stur wzr, [x29, #-4]
|
|
; CHECK-NEXT: sturh w8, [x29, #-8]
|
|
; CHECK-NEXT: msr TPIDR2_EL0, x9
|
|
; CHECK-NEXT: bl private_za_callee
|
|
; CHECK-NEXT: smstart za
|
|
; CHECK-NEXT: mrs x8, TPIDR2_EL0
|
|
; CHECK-NEXT: sub x0, x29, #16
|
|
; CHECK-NEXT: cbnz x8, .LBB0_2
|
|
; CHECK-NEXT: // %bb.1:
|
|
; CHECK-NEXT: bl __arm_tpidr2_restore
|
|
; CHECK-NEXT: .LBB0_2:
|
|
; CHECK-NEXT: msr TPIDR2_EL0, xzr
|
|
; CHECK-NEXT: mov sp, x29
|
|
; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
|
|
; CHECK-NEXT: ret
|
|
;
|
|
; CHECK-NEWLOWERING-LABEL: disable_tailcallopt:
|
|
; CHECK-NEWLOWERING: // %bb.0:
|
|
; CHECK-NEWLOWERING-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
|
|
; CHECK-NEWLOWERING-NEXT: mov x29, sp
|
|
; CHECK-NEWLOWERING-NEXT: sub sp, sp, #16
|
|
; CHECK-NEWLOWERING-NEXT: rdsvl x8, #1
|
|
; CHECK-NEWLOWERING-NEXT: mov x9, sp
|
|
; CHECK-NEWLOWERING-NEXT: msub x9, x8, x8, x9
|
|
; CHECK-NEWLOWERING-NEXT: mov sp, x9
|
|
; CHECK-NEWLOWERING-NEXT: sub x10, x29, #16
|
|
; CHECK-NEWLOWERING-NEXT: stp x9, x8, [x29, #-16]
|
|
; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, x10
|
|
; CHECK-NEWLOWERING-NEXT: bl private_za_callee
|
|
; CHECK-NEWLOWERING-NEXT: smstart za
|
|
; CHECK-NEWLOWERING-NEXT: mrs x8, TPIDR2_EL0
|
|
; CHECK-NEWLOWERING-NEXT: sub x0, x29, #16
|
|
; CHECK-NEWLOWERING-NEXT: cbnz x8, .LBB0_2
|
|
; CHECK-NEWLOWERING-NEXT: // %bb.1:
|
|
; CHECK-NEWLOWERING-NEXT: bl __arm_tpidr2_restore
|
|
; CHECK-NEWLOWERING-NEXT: .LBB0_2:
|
|
; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr
|
|
; CHECK-NEWLOWERING-NEXT: mov sp, x29
|
|
; CHECK-NEWLOWERING-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
|
|
; CHECK-NEWLOWERING-NEXT: ret
|
|
tail call void @private_za_callee()
|
|
ret void
|
|
}
|
|
|
|
; Ensure we set up and restore the lazy save correctly for instructions which are lowered to lib calls
|
|
define fp128 @f128_call_za(fp128 %a, fp128 %b) "aarch64_inout_za" nounwind {
|
|
; CHECK-LABEL: f128_call_za:
|
|
; CHECK: // %bb.0:
|
|
; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
|
|
; CHECK-NEXT: mov x29, sp
|
|
; CHECK-NEXT: sub sp, sp, #16
|
|
; CHECK-NEXT: rdsvl x8, #1
|
|
; CHECK-NEXT: mov x9, sp
|
|
; CHECK-NEXT: msub x9, x8, x8, x9
|
|
; CHECK-NEXT: mov sp, x9
|
|
; CHECK-NEXT: stur x9, [x29, #-16]
|
|
; CHECK-NEXT: sub x9, x29, #16
|
|
; CHECK-NEXT: sturh wzr, [x29, #-6]
|
|
; CHECK-NEXT: stur wzr, [x29, #-4]
|
|
; CHECK-NEXT: sturh w8, [x29, #-8]
|
|
; CHECK-NEXT: msr TPIDR2_EL0, x9
|
|
; CHECK-NEXT: bl __addtf3
|
|
; CHECK-NEXT: smstart za
|
|
; CHECK-NEXT: mrs x8, TPIDR2_EL0
|
|
; CHECK-NEXT: sub x0, x29, #16
|
|
; CHECK-NEXT: cbnz x8, .LBB1_2
|
|
; CHECK-NEXT: // %bb.1:
|
|
; CHECK-NEXT: bl __arm_tpidr2_restore
|
|
; CHECK-NEXT: .LBB1_2:
|
|
; CHECK-NEXT: msr TPIDR2_EL0, xzr
|
|
; CHECK-NEXT: mov sp, x29
|
|
; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
|
|
; CHECK-NEXT: ret
|
|
;
|
|
; CHECK-NEWLOWERING-LABEL: f128_call_za:
|
|
; CHECK-NEWLOWERING: // %bb.0:
|
|
; CHECK-NEWLOWERING-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
|
|
; CHECK-NEWLOWERING-NEXT: mov x29, sp
|
|
; CHECK-NEWLOWERING-NEXT: sub sp, sp, #16
|
|
; CHECK-NEWLOWERING-NEXT: rdsvl x8, #1
|
|
; CHECK-NEWLOWERING-NEXT: mov x9, sp
|
|
; CHECK-NEWLOWERING-NEXT: msub x9, x8, x8, x9
|
|
; CHECK-NEWLOWERING-NEXT: mov sp, x9
|
|
; CHECK-NEWLOWERING-NEXT: sub x10, x29, #16
|
|
; CHECK-NEWLOWERING-NEXT: stp x9, x8, [x29, #-16]
|
|
; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, x10
|
|
; CHECK-NEWLOWERING-NEXT: bl __addtf3
|
|
; CHECK-NEWLOWERING-NEXT: smstart za
|
|
; CHECK-NEWLOWERING-NEXT: mrs x8, TPIDR2_EL0
|
|
; CHECK-NEWLOWERING-NEXT: sub x0, x29, #16
|
|
; CHECK-NEWLOWERING-NEXT: cbnz x8, .LBB1_2
|
|
; CHECK-NEWLOWERING-NEXT: // %bb.1:
|
|
; CHECK-NEWLOWERING-NEXT: bl __arm_tpidr2_restore
|
|
; CHECK-NEWLOWERING-NEXT: .LBB1_2:
|
|
; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr
|
|
; CHECK-NEWLOWERING-NEXT: mov sp, x29
|
|
; CHECK-NEWLOWERING-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
|
|
; CHECK-NEWLOWERING-NEXT: ret
|
|
%res = fadd fp128 %a, %b
|
|
ret fp128 %res
|
|
}
|