This commit introduces attribute bpf_fastcall to declare BPF functions
that do not clobber some of the caller saved registers (R0-R5).
The idea is to generate the code complying with generic BPF ABI,
but allow compatible Linux Kernel to remove unnecessary spills and
fills of non-scratched registers (given some compiler assistance).
For such functions do register allocation as-if caller saved registers
are not clobbered, but later wrap the calls with spill and fill
patterns that are simple to recognize in kernel.
For example for the following C code:
#define __bpf_fastcall __attribute__((bpf_fastcall))
void bar(void) __bpf_fastcall;
void buz(long i, long j, long k);
void foo(long i, long j, long k) {
bar();
buz(i, j, k);
}
First allocate registers as if:
foo:
call bar # note: no spills for i,j,k (r1,r2,r3)
call buz
exit
And later insert spills fills on the peephole phase:
foo:
*(u64 *)(r10 - 8) = r1; # Such call pattern is
*(u64 *)(r10 - 16) = r2; # correct when used with
*(u64 *)(r10 - 24) = r3; # old kernels.
call bar
r3 = *(u64 *)(r10 - 24); # But also allows new
r2 = *(u64 *)(r10 - 16); # kernels to recognize the
r1 = *(u64 *)(r10 - 8); # pattern and remove spills/fills.
call buz
exit
The offsets for generated spills/fills are picked as minimal stack
offsets for the function. Allocated stack slots are not used for any
other purposes, in order to simplify in-kernel analysis.
148 lines
4.6 KiB
C++
148 lines
4.6 KiB
C++
//===-- BPFRegisterInfo.cpp - BPF Register Information ----------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file contains the BPF implementation of the TargetRegisterInfo class.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "BPFRegisterInfo.h"
|
|
#include "BPF.h"
|
|
#include "BPFSubtarget.h"
|
|
#include "llvm/CodeGen/MachineFrameInfo.h"
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
|
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
|
#include "llvm/CodeGen/RegisterScavenging.h"
|
|
#include "llvm/CodeGen/TargetFrameLowering.h"
|
|
#include "llvm/CodeGen/TargetInstrInfo.h"
|
|
#include "llvm/IR/DiagnosticInfo.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
#include "llvm/Support/ErrorHandling.h"
|
|
|
|
#define GET_REGINFO_TARGET_DESC
|
|
#include "BPFGenRegisterInfo.inc"
|
|
using namespace llvm;
|
|
|
|
static cl::opt<int>
|
|
BPFStackSizeOption("bpf-stack-size",
|
|
cl::desc("Specify the BPF stack size limit"),
|
|
cl::init(512));
|
|
|
|
BPFRegisterInfo::BPFRegisterInfo()
|
|
: BPFGenRegisterInfo(BPF::R0) {}
|
|
|
|
const MCPhysReg *
|
|
BPFRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
|
|
return CSR_SaveList;
|
|
}
|
|
|
|
const uint32_t *
|
|
BPFRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
|
|
CallingConv::ID CC) const {
|
|
switch (CC) {
|
|
default:
|
|
return CSR_RegMask;
|
|
case CallingConv::PreserveAll:
|
|
return CSR_PreserveAll_RegMask;
|
|
}
|
|
}
|
|
|
|
BitVector BPFRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
|
|
BitVector Reserved(getNumRegs());
|
|
markSuperRegs(Reserved, BPF::W10); // [W|R]10 is read only frame pointer
|
|
markSuperRegs(Reserved, BPF::W11); // [W|R]11 is pseudo stack pointer
|
|
return Reserved;
|
|
}
|
|
|
|
static void WarnSize(int Offset, MachineFunction &MF, DebugLoc& DL,
|
|
MachineBasicBlock& MBB) {
|
|
if (Offset <= -BPFStackSizeOption) {
|
|
if (!DL)
|
|
/* try harder to get some debug loc */
|
|
for (auto &I : MBB)
|
|
if (I.getDebugLoc()) {
|
|
DL = I.getDebugLoc();
|
|
break;
|
|
}
|
|
|
|
const Function &F = MF.getFunction();
|
|
DiagnosticInfoUnsupported DiagStackSize(
|
|
F,
|
|
"Looks like the BPF stack limit is exceeded. "
|
|
"Please move large on stack variables into BPF per-cpu array map. For "
|
|
"non-kernel uses, the stack can be increased using -mllvm "
|
|
"-bpf-stack-size.\n",
|
|
DL);
|
|
F.getContext().diagnose(DiagStackSize);
|
|
}
|
|
}
|
|
|
|
bool BPFRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
|
|
int SPAdj, unsigned FIOperandNum,
|
|
RegScavenger *RS) const {
|
|
assert(SPAdj == 0 && "Unexpected");
|
|
|
|
unsigned i = 0;
|
|
MachineInstr &MI = *II;
|
|
MachineBasicBlock &MBB = *MI.getParent();
|
|
MachineFunction &MF = *MBB.getParent();
|
|
DebugLoc DL = MI.getDebugLoc();
|
|
|
|
while (!MI.getOperand(i).isFI()) {
|
|
++i;
|
|
assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!");
|
|
}
|
|
|
|
Register FrameReg = getFrameRegister(MF);
|
|
int FrameIndex = MI.getOperand(i).getIndex();
|
|
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
|
|
|
|
if (MI.getOpcode() == BPF::MOV_rr) {
|
|
int Offset = MF.getFrameInfo().getObjectOffset(FrameIndex);
|
|
|
|
WarnSize(Offset, MF, DL, MBB);
|
|
MI.getOperand(i).ChangeToRegister(FrameReg, false);
|
|
Register reg = MI.getOperand(i - 1).getReg();
|
|
BuildMI(MBB, ++II, DL, TII.get(BPF::ADD_ri), reg)
|
|
.addReg(reg)
|
|
.addImm(Offset);
|
|
return false;
|
|
}
|
|
|
|
int Offset = MF.getFrameInfo().getObjectOffset(FrameIndex) +
|
|
MI.getOperand(i + 1).getImm();
|
|
|
|
if (!isInt<32>(Offset))
|
|
llvm_unreachable("bug in frame offset");
|
|
|
|
WarnSize(Offset, MF, DL, MBB);
|
|
|
|
if (MI.getOpcode() == BPF::FI_ri) {
|
|
// architecture does not really support FI_ri, replace it with
|
|
// MOV_rr <target_reg>, frame_reg
|
|
// ADD_ri <target_reg>, imm
|
|
Register reg = MI.getOperand(i - 1).getReg();
|
|
|
|
BuildMI(MBB, ++II, DL, TII.get(BPF::MOV_rr), reg)
|
|
.addReg(FrameReg);
|
|
BuildMI(MBB, II, DL, TII.get(BPF::ADD_ri), reg)
|
|
.addReg(reg)
|
|
.addImm(Offset);
|
|
|
|
// Remove FI_ri instruction
|
|
MI.eraseFromParent();
|
|
} else {
|
|
MI.getOperand(i).ChangeToRegister(FrameReg, false);
|
|
MI.getOperand(i + 1).ChangeToImmediate(Offset);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
Register BPFRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
|
|
return BPF::R10;
|
|
}
|