shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/lib/Target/AArch64/AArch64CallLowering.cpp
Jessica Paquette b1c1095fdc [AArch64][GlobalISel] Support lowering variadic musttail calls 
This adds support for lowering variadic musttail calls. To do this, we have
to...

- Detect a musttail call in a variadic function before attempting to lower the
  call's formal arguments. This is done in the IRTranslator.
- Compute forwarded registers in `lowerFormalArguments`, and add copies for
  those registers.
- Restore the forwarded registers in `lowerTailCall`.

Because there doesn't seem to be any nice way to wrap these up into the outgoing
argument handler, the restore code in `lowerTailCall` is done separately.

Also, irritatingly, you have to make sure that the registers don't overlap with
any passed parameters. Otherwise, the scheduler doesn't know what to do with the
extra copies and asserts.

Add call-translator-variadic-musttail.ll to test this. This is pretty much the
same as the X86 musttail-varargs.ll test. We didn't have as nice of a test to
base this off of, but the idea is the same.

Differential Revision: https://reviews.llvm.org/D68043

llvm-svn: 373226
2019-09-30 16:49:13 +00:00

1028 lines
39 KiB
C++
Raw Blame History

//===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file implements the lowering of LLVM calls to machine code calls for
/// GlobalISel.
///
//===----------------------------------------------------------------------===//
#include "AArch64CallLowering.h"
#include "AArch64ISelLowering.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64Subtarget.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/LowLevelType.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/MachineValueType.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#define DEBUG_TYPE "aarch64-call-lowering"
using namespace llvm;
AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
: CallLowering(&TLI) {}
namespace {
struct IncomingArgHandler : public CallLowering::ValueHandler {
IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
CCAssignFn *AssignFn)
: ValueHandler(MIRBuilder, MRI, AssignFn), StackUsed(0) {}
Register getStackAddress(uint64_t Size, int64_t Offset,
MachinePointerInfo &MPO) override {
auto &MFI = MIRBuilder.getMF().getFrameInfo();
int FI = MFI.CreateFixedObject(Size, Offset, true);
MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
Register AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 64));
MIRBuilder.buildFrameIndex(AddrReg, FI);
StackUsed = std::max(StackUsed, Size + Offset);
return AddrReg;
}
void assignValueToReg(Register ValVReg, Register PhysReg,
CCValAssign &VA) override {
markPhysRegUsed(PhysReg);
switch (VA.getLocInfo()) {
default:
MIRBuilder.buildCopy(ValVReg, PhysReg);
break;
case CCValAssign::LocInfo::SExt:
case CCValAssign::LocInfo::ZExt:
case CCValAssign::LocInfo::AExt: {
auto Copy = MIRBuilder.buildCopy(LLT{VA.getLocVT()}, PhysReg);
MIRBuilder.buildTrunc(ValVReg, Copy);
break;
}
}
}
void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
MachinePointerInfo &MPO, CCValAssign &VA) override {
// FIXME: Get alignment
auto MMO = MIRBuilder.getMF().getMachineMemOperand(
MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
1);
MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
}
/// How the physical register gets marked varies between formal
/// parameters (it's a basic-block live-in), and a call instruction
/// (it's an implicit-def of the BL).
virtual void markPhysRegUsed(unsigned PhysReg) = 0;
bool isIncomingArgumentHandler() const override { return true; }
uint64_t StackUsed;
};
struct FormalArgHandler : public IncomingArgHandler {
FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
CCAssignFn *AssignFn)
: IncomingArgHandler(MIRBuilder, MRI, AssignFn) {}
void markPhysRegUsed(unsigned PhysReg) override {
MIRBuilder.getMRI()->addLiveIn(PhysReg);
MIRBuilder.getMBB().addLiveIn(PhysReg);
}
};
struct CallReturnHandler : public IncomingArgHandler {
CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
MachineInstrBuilder MIB, CCAssignFn *AssignFn)
: IncomingArgHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
void markPhysRegUsed(unsigned PhysReg) override {
MIB.addDef(PhysReg, RegState::Implicit);
}
MachineInstrBuilder MIB;
};
struct OutgoingArgHandler : public CallLowering::ValueHandler {
OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
MachineInstrBuilder MIB, CCAssignFn *AssignFn,
CCAssignFn *AssignFnVarArg, bool IsTailCall = false,
int FPDiff = 0)
: ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
AssignFnVarArg(AssignFnVarArg), IsTailCall(IsTailCall), FPDiff(FPDiff),
StackSize(0) {}
Register getStackAddress(uint64_t Size, int64_t Offset,
MachinePointerInfo &MPO) override {
MachineFunction &MF = MIRBuilder.getMF();
LLT p0 = LLT::pointer(0, 64);
LLT s64 = LLT::scalar(64);
if (IsTailCall) {
Offset += FPDiff;
int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
Register FIReg = MRI.createGenericVirtualRegister(p0);
MIRBuilder.buildFrameIndex(FIReg, FI);
MPO = MachinePointerInfo::getFixedStack(MF, FI);
return FIReg;
}
Register SPReg = MRI.createGenericVirtualRegister(p0);
MIRBuilder.buildCopy(SPReg, Register(AArch64::SP));
Register OffsetReg = MRI.createGenericVirtualRegister(s64);
MIRBuilder.buildConstant(OffsetReg, Offset);
Register AddrReg = MRI.createGenericVirtualRegister(p0);
MIRBuilder.buildGEP(AddrReg, SPReg, OffsetReg);
MPO = MachinePointerInfo::getStack(MF, Offset);
return AddrReg;
}
void assignValueToReg(Register ValVReg, Register PhysReg,
CCValAssign &VA) override {
MIB.addUse(PhysReg, RegState::Implicit);
Register ExtReg = extendRegister(ValVReg, VA);
MIRBuilder.buildCopy(PhysReg, ExtReg);
}
void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
MachinePointerInfo &MPO, CCValAssign &VA) override {
if (VA.getLocInfo() == CCValAssign::LocInfo::AExt) {
Size = VA.getLocVT().getSizeInBits() / 8;
ValVReg = MIRBuilder.buildAnyExt(LLT::scalar(Size * 8), ValVReg)
->getOperand(0)
.getReg();
}
auto MMO = MIRBuilder.getMF().getMachineMemOperand(
MPO, MachineMemOperand::MOStore, Size, 1);
MIRBuilder.buildStore(ValVReg, Addr, *MMO);
}
bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT,
CCValAssign::LocInfo LocInfo,
const CallLowering::ArgInfo &Info,
ISD::ArgFlagsTy Flags,
CCState &State) override {
bool Res;
if (Info.IsFixed)
Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
else
Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, State);
StackSize = State.getNextStackOffset();
return Res;
}
MachineInstrBuilder MIB;
CCAssignFn *AssignFnVarArg;
bool IsTailCall;
/// For tail calls, the byte offset of the call's argument area from the
/// callee's. Unused elsewhere.
int FPDiff;
uint64_t StackSize;
};
} // namespace
static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt) {
return CallConv == CallingConv::Fast && TailCallOpt;
}
void AArch64CallLowering::splitToValueTypes(
const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs,
const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv) const {
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
LLVMContext &Ctx = OrigArg.Ty->getContext();
if (OrigArg.Ty->isVoidTy())
return;
SmallVector<EVT, 4> SplitVTs;
SmallVector<uint64_t, 4> Offsets;
ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0);
if (SplitVTs.size() == 1) {
// No splitting to do, but we want to replace the original type (e.g. [1 x
// double] -> double).
SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
OrigArg.Flags[0], OrigArg.IsFixed);
return;
}
// Create one ArgInfo for each virtual register in the original ArgInfo.
assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
OrigArg.Ty, CallConv, false);
for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.Flags[0],
OrigArg.IsFixed);
if (NeedsRegBlock)
SplitArgs.back().Flags[0].setInConsecutiveRegs();
}
SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
}
bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
const Value *Val,
ArrayRef<Register> VRegs,
Register SwiftErrorVReg) const {
auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR);
assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&
"Return value without a vreg");
bool Success = true;
if (!VRegs.empty()) {
MachineFunction &MF = MIRBuilder.getMF();
const Function &F = MF.getFunction();
MachineRegisterInfo &MRI = MF.getRegInfo();
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
auto &DL = F.getParent()->getDataLayout();
LLVMContext &Ctx = Val->getType()->getContext();
SmallVector<EVT, 4> SplitEVTs;
ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
assert(VRegs.size() == SplitEVTs.size() &&
"For each split Type there should be exactly one VReg.");
SmallVector<ArgInfo, 8> SplitArgs;
CallingConv::ID CC = F.getCallingConv();
for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
if (TLI.getNumRegistersForCallingConv(Ctx, CC, SplitEVTs[i]) > 1) {
LLVM_DEBUG(dbgs() << "Can't handle extended arg types which need split");
return false;
}
Register CurVReg = VRegs[i];
ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx)};
setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F);
// i1 is a special case because SDAG i1 true is naturally zero extended
// when widened using ANYEXT. We need to do it explicitly here.
if (MRI.getType(CurVReg).getSizeInBits() == 1) {
CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg).getReg(0);
} else {
// Some types will need extending as specified by the CC.
MVT NewVT = TLI.getRegisterTypeForCallingConv(Ctx, CC, SplitEVTs[i]);
if (EVT(NewVT) != SplitEVTs[i]) {
unsigned ExtendOp = TargetOpcode::G_ANYEXT;
if (F.getAttributes().hasAttribute(AttributeList::ReturnIndex,
Attribute::SExt))
ExtendOp = TargetOpcode::G_SEXT;
else if (F.getAttributes().hasAttribute(AttributeList::ReturnIndex,
Attribute::ZExt))
ExtendOp = TargetOpcode::G_ZEXT;
LLT NewLLT(NewVT);
LLT OldLLT(MVT::getVT(CurArgInfo.Ty));
CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Ctx);
// Instead of an extend, we might have a vector type which needs
// padding with more elements, e.g. <2 x half> -> <4 x half>.
if (NewVT.isVector()) {
if (OldLLT.isVector()) {
if (NewLLT.getNumElements() > OldLLT.getNumElements()) {
// We don't handle VA types which are not exactly twice the
// size, but can easily be done in future.
if (NewLLT.getNumElements() != OldLLT.getNumElements() * 2) {
LLVM_DEBUG(dbgs() << "Outgoing vector ret has too many elts");
return false;
}
auto Undef = MIRBuilder.buildUndef({OldLLT});
CurVReg =
MIRBuilder.buildMerge({NewLLT}, {CurVReg, Undef.getReg(0)})
.getReg(0);
} else {
// Just do a vector extend.
CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
.getReg(0);
}
} else if (NewLLT.getNumElements() == 2) {
// We need to pad a <1 x S> type to <2 x S>. Since we don't have
// <1 x S> vector types in GISel we use a build_vector instead
// of a vector merge/concat.
auto Undef = MIRBuilder.buildUndef({OldLLT});
CurVReg =
MIRBuilder
.buildBuildVector({NewLLT}, {CurVReg, Undef.getReg(0)})
.getReg(0);
} else {
LLVM_DEBUG(dbgs() << "Could not handle ret ty");
return false;
}
} else {
// A scalar extend.
CurVReg =
MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg}).getReg(0);
}
}
}
if (CurVReg != CurArgInfo.Regs[0]) {
CurArgInfo.Regs[0] = CurVReg;
// Reset the arg flags after modifying CurVReg.
setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F);
}
splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, CC);
}
OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFn, AssignFn);
Success = handleAssignments(MIRBuilder, SplitArgs, Handler);
}
if (SwiftErrorVReg) {
MIB.addUse(AArch64::X21, RegState::Implicit);
MIRBuilder.buildCopy(AArch64::X21, SwiftErrorVReg);
}
MIRBuilder.insertInstr(MIB);
return Success;
}
/// Helper function to compute forwarded registers for musttail calls. Computes
/// the forwarded registers, sets MBB liveness, and emits COPY instructions that
/// can be used to save + restore registers later.
static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
CCAssignFn *AssignFn) {
MachineBasicBlock &MBB = MIRBuilder.getMBB();
MachineFunction &MF = MIRBuilder.getMF();
MachineFrameInfo &MFI = MF.getFrameInfo();
if (!MFI.hasMustTailInVarArgFunc())
return;
AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
const Function &F = MF.getFunction();
assert(F.isVarArg() && "Expected F to be vararg?");
// Compute the set of forwarded registers. The rest are scratch.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(F.getCallingConv(), /*IsVarArg=*/true, MF, ArgLocs,
F.getContext());
SmallVector<MVT, 2> RegParmTypes;
RegParmTypes.push_back(MVT::i64);
RegParmTypes.push_back(MVT::f128);
// Later on, we can use this vector to restore the registers if necessary.
SmallVectorImpl<ForwardedRegister> &Forwards =
FuncInfo->getForwardedMustTailRegParms();
CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, AssignFn);
// Conservatively forward X8, since it might be used for an aggregate
// return.
if (!CCInfo.isAllocated(AArch64::X8)) {
unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
}
// Add the forwards to the MachineBasicBlock and MachineFunction.
for (const auto &F : Forwards) {
MBB.addLiveIn(F.PReg);
MIRBuilder.buildCopy(Register(F.VReg), Register(F.PReg));
}
}
bool AArch64CallLowering::lowerFormalArguments(
MachineIRBuilder &MIRBuilder, const Function &F,
ArrayRef<ArrayRef<Register>> VRegs) const {
MachineFunction &MF = MIRBuilder.getMF();
MachineBasicBlock &MBB = MIRBuilder.getMBB();
MachineRegisterInfo &MRI = MF.getRegInfo();
auto &DL = F.getParent()->getDataLayout();
SmallVector<ArgInfo, 8> SplitArgs;
unsigned i = 0;
for (auto &Arg : F.args()) {
if (DL.getTypeStoreSize(Arg.getType()) == 0)
continue;
ArgInfo OrigArg{VRegs[i], Arg.getType()};
setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
splitToValueTypes(OrigArg, SplitArgs, DL, MRI, F.getCallingConv());
++i;
}
if (!MBB.empty())
MIRBuilder.setInstr(*MBB.begin());
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
CCAssignFn *AssignFn =
TLI.CCAssignFnForCall(F.getCallingConv(), /*IsVarArg=*/false);
FormalArgHandler Handler(MIRBuilder, MRI, AssignFn);
if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
return false;
AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
uint64_t StackOffset = Handler.StackUsed;
if (F.isVarArg()) {
auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
if (!Subtarget.isTargetDarwin()) {
// FIXME: we need to reimplement saveVarArgsRegisters from
// AArch64ISelLowering.
return false;
}
// We currently pass all varargs at 8-byte alignment, or 4 in ILP32.
StackOffset = alignTo(Handler.StackUsed, Subtarget.isTargetILP32() ? 4 : 8);
auto &MFI = MIRBuilder.getMF().getFrameInfo();
FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
}
if (doesCalleeRestoreStack(F.getCallingConv(),
MF.getTarget().Options.GuaranteedTailCallOpt)) {
// We have a non-standard ABI, so why not make full use of the stack that
// we're going to pop? It must be aligned to 16 B in any case.
StackOffset = alignTo(StackOffset, 16);
// If we're expected to restore the stack (e.g. fastcc), then we'll be
// adding a multiple of 16.
FuncInfo->setArgumentStackToRestore(StackOffset);
// Our own callers will guarantee that the space is free by giving an
// aligned value to CALLSEQ_START.
}
// When we tail call, we need to check if the callee's arguments
// will fit on the caller's stack. So, whenever we lower formal arguments,
// we should keep track of this information, since we might lower a tail call
// in this function later.
FuncInfo->setBytesInStackArgArea(StackOffset);
auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
if (Subtarget.hasCustomCallingConv())
Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
handleMustTailForwardedRegisters(MIRBuilder, AssignFn);
// Move back to the end of the basic block.
MIRBuilder.setMBB(MBB);
return true;
}
/// Return true if the calling convention is one that we can guarantee TCO for.
static bool canGuaranteeTCO(CallingConv::ID CC) {
return CC == CallingConv::Fast;
}
/// Return true if we might ever do TCO for calls with this calling convention.
static bool mayTailCallThisCC(CallingConv::ID CC) {
switch (CC) {
case CallingConv::C:
case CallingConv::PreserveMost:
case CallingConv::Swift:
return true;
default:
return canGuaranteeTCO(CC);
}
}
/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
/// CC.
static std::pair<CCAssignFn *, CCAssignFn *>
getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI) {
return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
}
bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
CallLoweringInfo &Info, MachineFunction &MF,
SmallVectorImpl<ArgInfo> &InArgs) const {
const Function &CallerF = MF.getFunction();
CallingConv::ID CalleeCC = Info.CallConv;
CallingConv::ID CallerCC = CallerF.getCallingConv();
// If the calling conventions match, then everything must be the same.
if (CalleeCC == CallerCC)
return true;
// Check if the caller and callee will handle arguments in the same way.
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
CCAssignFn *CalleeAssignFnFixed;
CCAssignFn *CalleeAssignFnVarArg;
std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
getAssignFnsForCC(CalleeCC, TLI);
CCAssignFn *CallerAssignFnFixed;
CCAssignFn *CallerAssignFnVarArg;
std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
getAssignFnsForCC(CallerCC, TLI);
if (!resultsCompatible(Info, MF, InArgs, *CalleeAssignFnFixed,
*CalleeAssignFnVarArg, *CallerAssignFnFixed,
*CallerAssignFnVarArg))
return false;
// Make sure that the caller and callee preserve all of the same registers.
auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv()) {
TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
}
return TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved);
}
bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
CallLoweringInfo &Info, MachineFunction &MF,
SmallVectorImpl<ArgInfo> &OutArgs) const {
// If there are no outgoing arguments, then we are done.
if (OutArgs.empty())
return true;
const Function &CallerF = MF.getFunction();
CallingConv::ID CalleeCC = Info.CallConv;
CallingConv::ID CallerCC = CallerF.getCallingConv();
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
CCAssignFn *AssignFnFixed;
CCAssignFn *AssignFnVarArg;
std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
// We have outgoing arguments. Make sure that we can tail call with them.
SmallVector<CCValAssign, 16> OutLocs;
CCState OutInfo(CalleeCC, false, MF, OutLocs, CallerF.getContext());
if (!analyzeArgInfo(OutInfo, OutArgs, *AssignFnFixed, *AssignFnVarArg)) {
LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
return false;
}
// Make sure that they can fit on the caller's stack.
const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
if (OutInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea()) {
LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");
return false;
}
// Verify that the parameters in callee-saved registers match.
// TODO: Port this over to CallLowering as general code once swiftself is
// supported.
auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
MachineRegisterInfo &MRI = MF.getRegInfo();
for (unsigned i = 0; i < OutLocs.size(); ++i) {
auto &ArgLoc = OutLocs[i];
// If it's not a register, it's fine.
if (!ArgLoc.isRegLoc()) {
if (Info.IsVarArg) {
// Be conservative and disallow variadic memory operands to match SDAG's
// behaviour.
// FIXME: If the caller's calling convention is C, then we can
// potentially use its argument area. However, for cases like fastcc,
// we can't do anything.
LLVM_DEBUG(
dbgs()
<< "... Cannot tail call vararg function with stack arguments\n");
return false;
}
continue;
}
Register Reg = ArgLoc.getLocReg();
// Only look at callee-saved registers.
if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
continue;
LLVM_DEBUG(
dbgs()
<< "... Call has an argument passed in a callee-saved register.\n");
// Check if it was copied from.
ArgInfo &OutInfo = OutArgs[i];
if (OutInfo.Regs.size() > 1) {
LLVM_DEBUG(
dbgs() << "... Cannot handle arguments in multiple registers.\n");
return false;
}
// Check if we copy the register, walking through copies from virtual
// registers. Note that getDefIgnoringCopies does not ignore copies from
// physical registers.
MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);
if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
LLVM_DEBUG(
dbgs()
<< "... Parameter was not copied into a VReg, cannot tail call.\n");
return false;
}
// Got a copy. Verify that it's the same as the register we want.
Register CopyRHS = RegDef->getOperand(1).getReg();
if (CopyRHS != Reg) {
LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
"VReg, cannot tail call.\n");
return false;
}
}
return true;
}
bool AArch64CallLowering::isEligibleForTailCallOptimization(
MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
SmallVectorImpl<ArgInfo> &InArgs,
SmallVectorImpl<ArgInfo> &OutArgs) const {
// Must pass all target-independent checks in order to tail call optimize.
if (!Info.IsTailCall)
return false;
CallingConv::ID CalleeCC = Info.CallConv;
MachineFunction &MF = MIRBuilder.getMF();
const Function &CallerF = MF.getFunction();
LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n");
if (Info.SwiftErrorVReg) {
// TODO: We should handle this.
// Note that this is also handled by the check for no outgoing arguments.
// Proactively disabling this though, because the swifterror handling in
// lowerCall inserts a COPY *after* the location of the call.
LLVM_DEBUG(dbgs() << "... Cannot handle tail calls with swifterror yet.\n");
return false;
}
if (!mayTailCallThisCC(CalleeCC)) {
LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
return false;
}
// Byval parameters hand the function a pointer directly into the stack area
// we want to reuse during a tail call. Working around this *is* possible (see
// X86).
//
// FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
// it?
//
// On Windows, "inreg" attributes signify non-aggregate indirect returns.
// In this case, it is necessary to save/restore X0 in the callee. Tail
// call opt interferes with this. So we disable tail call opt when the
// caller has an argument with "inreg" attribute.
//
// FIXME: Check whether the callee also has an "inreg" argument.
//
// When the caller has a swifterror argument, we don't want to tail call
// because would have to move into the swifterror register before the
// tail call.
if (any_of(CallerF.args(), [](const Argument &A) {
return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
})) {
LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "
"inreg, or swifterror arguments\n");
return false;
}
// Externally-defined functions with weak linkage should not be
// tail-called on AArch64 when the OS does not support dynamic
// pre-emption of symbols, as the AAELF spec requires normal calls
// to undefined weak functions to be replaced with a NOP or jump to the
// next instruction. The behaviour of branch instructions in this
// situation (as used for tail calls) is implementation-defined, so we
// cannot rely on the linker replacing the tail call with a return.
if (Info.Callee.isGlobal()) {
const GlobalValue *GV = Info.Callee.getGlobal();
const Triple &TT = MF.getTarget().getTargetTriple();
if (GV->hasExternalWeakLinkage() &&
(!TT.isOSWindows() || TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO())) {
LLVM_DEBUG(dbgs() << "... Cannot tail call externally-defined function "
"with weak linkage for this OS.\n");
return false;
}
}
// If we have -tailcallopt, then we're done.
if (MF.getTarget().Options.GuaranteedTailCallOpt)
return canGuaranteeTCO(CalleeCC) && CalleeCC == CallerF.getCallingConv();
// We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
// Try to find cases where we can do that.
// I want anyone implementing a new calling convention to think long and hard
// about this assert.
assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&
"Unexpected variadic calling convention");
// Verify that the incoming and outgoing arguments from the callee are
// safe to tail call.
if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
LLVM_DEBUG(
dbgs()
<< "... Caller and callee have incompatible calling conventions.\n");
return false;
}
if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
return false;
LLVM_DEBUG(
dbgs() << "... Call is eligible for tail call optimization.\n");
return true;
}
static unsigned getCallOpcode(const Function &CallerF, bool IsIndirect,
bool IsTailCall) {
if (!IsTailCall)
return IsIndirect ? AArch64::BLR : AArch64::BL;
if (!IsIndirect)
return AArch64::TCRETURNdi;
// When BTI is enabled, we need to use TCRETURNriBTI to make sure that we use
// x16 or x17.
if (CallerF.hasFnAttribute("branch-target-enforcement"))
return AArch64::TCRETURNriBTI;
return AArch64::TCRETURNri;
}
bool AArch64CallLowering::lowerTailCall(
MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
SmallVectorImpl<ArgInfo> &OutArgs) const {
MachineFunction &MF = MIRBuilder.getMF();
const Function &F = MF.getFunction();
MachineRegisterInfo &MRI = MF.getRegInfo();
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
// True when we're tail calling, but without -tailcallopt.
bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt;
// TODO: Right now, regbankselect doesn't know how to handle the rtcGPR64
// register class. Until we can do that, we should fall back here.
if (F.hasFnAttribute("branch-target-enforcement")) {
LLVM_DEBUG(
dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n");
return false;
}
// Find out which ABI gets to decide where things go.
CallingConv::ID CalleeCC = Info.CallConv;
CCAssignFn *AssignFnFixed;
CCAssignFn *AssignFnVarArg;
std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
MachineInstrBuilder CallSeqStart;
if (!IsSibCall)
CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
unsigned Opc = getCallOpcode(F, Info.Callee.isReg(), true);
auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
MIB.add(Info.Callee);
// Byte offset for the tail call. When we are sibcalling, this will always
// be 0.
MIB.addImm(0);
// Tell the call which registers are clobbered.
auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
const uint32_t *Mask = TRI->getCallPreservedMask(MF, F.getCallingConv());
if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
TRI->UpdateCustomCallPreservedMask(MF, &Mask);
MIB.addRegMask(Mask);
if (TRI->isAnyArgRegReserved(MF))
TRI->emitReservedArgRegCallError(MF);
// FPDiff is the byte offset of the call's argument area from the callee's.
// Stores to callee stack arguments will be placed in FixedStackSlots offset
// by this amount for a tail call. In a sibling call it must be 0 because the
// caller will deallocate the entire stack and the callee still expects its
// arguments to begin at SP+0.
int FPDiff = 0;
// This will be 0 for sibcalls, potentially nonzero for tail calls produced
// by -tailcallopt. For sibcalls, the memory operands for the call are
// already available in the caller's incoming argument space.
unsigned NumBytes = 0;
if (!IsSibCall) {
// We aren't sibcalling, so we need to compute FPDiff. We need to do this
// before handling assignments, because FPDiff must be known for memory
// arguments.
unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
SmallVector<CCValAssign, 16> OutLocs;
CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
analyzeArgInfo(OutInfo, OutArgs, *AssignFnFixed, *AssignFnVarArg);
// The callee will pop the argument stack as a tail call. Thus, we must
// keep it 16-byte aligned.
NumBytes = alignTo(OutInfo.getNextStackOffset(), 16);
// FPDiff will be negative if this tail call requires more space than we
// would automatically have in our incoming argument space. Positive if we
// actually shrink the stack.
FPDiff = NumReusableBytes - NumBytes;
// The stack pointer must be 16-byte aligned at all times it's used for a
// memory operation, which in practice means at *all* times and in
// particular across call boundaries. Therefore our own arguments started at
// a 16-byte aligned SP and the delta applied for the tail call should
// satisfy the same constraint.
assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
}
const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
// Do the actual argument marshalling.
SmallVector<unsigned, 8> PhysRegs;
OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
AssignFnVarArg, true, FPDiff);
if (!handleAssignments(MIRBuilder, OutArgs, Handler))
return false;
if (Info.IsVarArg && Info.IsMustTailCall) {
// Now we know what's being passed to the function. Add uses to the call for
// the forwarded registers that we *aren't* passing as parameters. This will
// preserve the copies we build earlier.
for (const auto &F : Forwards) {
Register ForwardedReg = F.PReg;
// If the register is already passed, or aliases a register which is
// already being passed, then skip it.
if (any_of(MIB->uses(), [&ForwardedReg, &TRI](const MachineOperand &Use) {
if (!Use.isReg())
return false;
return TRI->regsOverlap(Use.getReg(), ForwardedReg);
}))
continue;
// We aren't passing it already, so we should add it to the call.
MIRBuilder.buildCopy(ForwardedReg, Register(F.VReg));
MIB.addReg(ForwardedReg, RegState::Implicit);
}
}
// If we have -tailcallopt, we need to adjust the stack. We'll do the call
// sequence start and end here.
if (!IsSibCall) {
MIB->getOperand(1).setImm(FPDiff);
CallSeqStart.addImm(NumBytes).addImm(0);
// End the call sequence *before* emitting the call. Normally, we would
// tidy the frame up after the call. However, here, we've laid out the
// parameters so that when SP is reset, they will be in the correct
// location.
MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP).addImm(NumBytes).addImm(0);
}
// Now we can add the actual call instruction to the correct basic block.
MIRBuilder.insertInstr(MIB);
// If Callee is a reg, since it is used by a target specific instruction,
// it must have a register class matching the constraint of that instruction.
if (Info.Callee.isReg())
MIB->getOperand(0).setReg(constrainOperandRegClass(
MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
*MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee,
0));
MF.getFrameInfo().setHasTailCall();
Info.LoweredTailCall = true;
return true;
}
bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
CallLoweringInfo &Info) const {
MachineFunction &MF = MIRBuilder.getMF();
const Function &F = MF.getFunction();
MachineRegisterInfo &MRI = MF.getRegInfo();
auto &DL = F.getParent()->getDataLayout();
const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
SmallVector<ArgInfo, 8> OutArgs;
for (auto &OrigArg : Info.OrigArgs) {
splitToValueTypes(OrigArg, OutArgs, DL, MRI, Info.CallConv);
// AAPCS requires that we zero-extend i1 to 8 bits by the caller.
if (OrigArg.Ty->isIntegerTy(1))
OutArgs.back().Flags[0].setZExt();
}
SmallVector<ArgInfo, 8> InArgs;
if (!Info.OrigRet.Ty->isVoidTy())
splitToValueTypes(Info.OrigRet, InArgs, DL, MRI, F.getCallingConv());
// If we can lower as a tail call, do that instead.
bool CanTailCallOpt =
isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
// We must emit a tail call if we have musttail.
if (Info.IsMustTailCall && !CanTailCallOpt) {
// There are types of incoming/outgoing arguments we can't handle yet, so
// it doesn't make sense to actually die here like in ISelLowering. Instead,
// fall back to SelectionDAG and let it try to handle this.
LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
return false;
}
if (CanTailCallOpt)
return lowerTailCall(MIRBuilder, Info, OutArgs);
// Find out which ABI gets to decide where things go.
CCAssignFn *AssignFnFixed;
CCAssignFn *AssignFnVarArg;
std::tie(AssignFnFixed, AssignFnVarArg) =
getAssignFnsForCC(Info.CallConv, TLI);
MachineInstrBuilder CallSeqStart;
CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
// Create a temporarily-floating call instruction so we can add the implicit
// uses of arg registers.
unsigned Opc = getCallOpcode(F, Info.Callee.isReg(), false);
auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
MIB.add(Info.Callee);
// Tell the call which registers are clobbered.
auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
const uint32_t *Mask = TRI->getCallPreservedMask(MF, F.getCallingConv());
if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
TRI->UpdateCustomCallPreservedMask(MF, &Mask);
MIB.addRegMask(Mask);
if (TRI->isAnyArgRegReserved(MF))
TRI->emitReservedArgRegCallError(MF);
// Do the actual argument marshalling.
SmallVector<unsigned, 8> PhysRegs;
OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
AssignFnVarArg, false);
if (!handleAssignments(MIRBuilder, OutArgs, Handler))
return false;
// Now we can add the actual call instruction to the correct basic block.
MIRBuilder.insertInstr(MIB);
// If Callee is a reg, since it is used by a target specific
// instruction, it must have a register class matching the
// constraint of that instruction.
if (Info.Callee.isReg())
MIB->getOperand(0).setReg(constrainOperandRegClass(
MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
*MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee,
0));
// Finally we can copy the returned value back into its virtual-register. In
// symmetry with the arugments, the physical register must be an
// implicit-define of the call instruction.
if (!Info.OrigRet.Ty->isVoidTy()) {
CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
CallReturnHandler Handler(MIRBuilder, MRI, MIB, RetAssignFn);
if (!handleAssignments(MIRBuilder, InArgs, Handler))
return false;
}
if (Info.SwiftErrorVReg) {
MIB.addDef(AArch64::X21, RegState::Implicit);
MIRBuilder.buildCopy(Info.SwiftErrorVReg, Register(AArch64::X21));
}
uint64_t CalleePopBytes =
doesCalleeRestoreStack(Info.CallConv,
MF.getTarget().Options.GuaranteedTailCallOpt)
? alignTo(Handler.StackSize, 16)
: 0;
CallSeqStart.addImm(Handler.StackSize).addImm(0);
MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
.addImm(Handler.StackSize)
.addImm(CalleePopBytes);
return true;
}
Reference in New Issue View Git Blame Copy Permalink