2aacb56e83
This commit aims to support BPF arena kernel side [feature](https://lore.kernel.org/bpf/20240209040608.98927-1-alexei.starovoitov@gmail.com/): - arena is a memory region accessible from both BPF program and userspace; - base pointers for this memory region differ between kernel and user spaces; - `dst_reg = addr_space_cast(src_reg, dst_addr_space, src_addr_space)` translates src_reg, a pointer in src_addr_space to dst_reg, equivalent pointer in dst_addr_space, {src,dst}_addr_space are immediate constants; - number 0 is assigned to kernel address space; - number 1 is assigned to user address space. On the LLVM side, the goal is to make load and store operations on arena pointers "transparent" for BPF programs: - assume that pointers with non-zero address space are pointers to arena memory; - assume that arena is identified by address space number; - assume that address space zero corresponds to kernel address space; - assume that every BPF-side load or store from arena is done via pointer in user address space, thus convert base pointers using `addr_space_cast(src_reg, 0, 1)`; Only load, store, cmpxchg and atomicrmw IR instructions are handled by this transformation. For example, the following C code: ```c #define __as __attribute__((address_space(1))) void copy(int __as *from, int __as *to) { *to = *from; } ``` Compiled to the following IR: ```llvm define void @copy(ptr addrspace(1) %from, ptr addrspace(1) %to) { entry: %0 = load i32, ptr addrspace(1) %from, align 4 store i32 %0, ptr addrspace(1) %to, align 4 ret void } ``` Is transformed to: ```llvm %to2 = addrspacecast ptr addrspace(1) %to to ptr ;; ! %from1 = addrspacecast ptr addrspace(1) %from to ptr ;; ! %0 = load i32, ptr %from1, align 4, !tbaa !3 store i32 %0, ptr %to2, align 4, !tbaa !3 ret void ``` And compiled as: ```asm r2 = addr_space_cast(r2, 0, 1) r1 = addr_space_cast(r1, 0, 1) r1 = *(u32 *)(r1 + 0) *(u32 *)(r2 + 0) = r1 exit ``` Co-authored-by: Eduard Zingerman <eddyz87@gmail.com>
93 lines
3.1 KiB
C++
93 lines
3.1 KiB
C++
//===-- BPFASpaceCastSimplifyPass.cpp - BPF addrspacecast simplications --===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "BPF.h"
|
|
#include <optional>
|
|
|
|
#define DEBUG_TYPE "bpf-aspace-simplify"
|
|
|
|
using namespace llvm;
|
|
|
|
namespace {
|
|
|
|
struct CastGEPCast {
|
|
AddrSpaceCastInst *OuterCast;
|
|
|
|
// Match chain of instructions:
|
|
// %inner = addrspacecast N->M
|
|
// %gep = getelementptr %inner, ...
|
|
// %outer = addrspacecast M->N %gep
|
|
// Where I is %outer.
|
|
static std::optional<CastGEPCast> match(Value *I) {
|
|
auto *OuterCast = dyn_cast<AddrSpaceCastInst>(I);
|
|
if (!OuterCast)
|
|
return std::nullopt;
|
|
auto *GEP = dyn_cast<GetElementPtrInst>(OuterCast->getPointerOperand());
|
|
if (!GEP)
|
|
return std::nullopt;
|
|
auto *InnerCast = dyn_cast<AddrSpaceCastInst>(GEP->getPointerOperand());
|
|
if (!InnerCast)
|
|
return std::nullopt;
|
|
if (InnerCast->getSrcAddressSpace() != OuterCast->getDestAddressSpace())
|
|
return std::nullopt;
|
|
if (InnerCast->getDestAddressSpace() != OuterCast->getSrcAddressSpace())
|
|
return std::nullopt;
|
|
return CastGEPCast{OuterCast};
|
|
}
|
|
|
|
static PointerType *changeAddressSpace(PointerType *Ty, unsigned AS) {
|
|
return Ty->get(Ty->getContext(), AS);
|
|
}
|
|
|
|
// Assuming match(this->OuterCast) is true, convert:
|
|
// (addrspacecast M->N (getelementptr (addrspacecast N->M ptr) ...))
|
|
// To:
|
|
// (getelementptr ptr ...)
|
|
GetElementPtrInst *rewrite() {
|
|
auto *GEP = cast<GetElementPtrInst>(OuterCast->getPointerOperand());
|
|
auto *InnerCast = cast<AddrSpaceCastInst>(GEP->getPointerOperand());
|
|
unsigned AS = OuterCast->getDestAddressSpace();
|
|
auto *NewGEP = cast<GetElementPtrInst>(GEP->clone());
|
|
NewGEP->setName(GEP->getName());
|
|
NewGEP->insertAfter(OuterCast);
|
|
NewGEP->setOperand(0, InnerCast->getPointerOperand());
|
|
auto *GEPTy = cast<PointerType>(GEP->getType());
|
|
NewGEP->mutateType(changeAddressSpace(GEPTy, AS));
|
|
OuterCast->replaceAllUsesWith(NewGEP);
|
|
OuterCast->eraseFromParent();
|
|
if (GEP->use_empty())
|
|
GEP->eraseFromParent();
|
|
if (InnerCast->use_empty())
|
|
InnerCast->eraseFromParent();
|
|
return NewGEP;
|
|
}
|
|
};
|
|
|
|
} // anonymous namespace
|
|
|
|
PreservedAnalyses BPFASpaceCastSimplifyPass::run(Function &F,
|
|
FunctionAnalysisManager &AM) {
|
|
SmallVector<CastGEPCast, 16> WorkList;
|
|
bool Changed = false;
|
|
for (BasicBlock &BB : F) {
|
|
for (Instruction &I : BB)
|
|
if (auto It = CastGEPCast::match(&I))
|
|
WorkList.push_back(It.value());
|
|
Changed |= !WorkList.empty();
|
|
|
|
while (!WorkList.empty()) {
|
|
CastGEPCast InsnChain = WorkList.pop_back_val();
|
|
GetElementPtrInst *NewGEP = InsnChain.rewrite();
|
|
for (User *U : NewGEP->users())
|
|
if (auto It = CastGEPCast::match(U))
|
|
WorkList.push_back(It.value());
|
|
}
|
|
}
|
|
return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
|
|
}
|