17bfc00f7c
As discussed in [1], introduce BPF instructions with load-acquire and
store-release semantics under -mcpu=v4. Define 2 new flags:
BPF_LOAD_ACQ 0x100
BPF_STORE_REL 0x110
A "load-acquire" is a BPF_STX | BPF_ATOMIC instruction with the 'imm'
field set to BPF_LOAD_ACQ (0x100).
Similarly, a "store-release" is a BPF_STX | BPF_ATOMIC instruction with
the 'imm' field set to BPF_STORE_REL (0x110).
Unlike existing atomic read-modify-write operations that only support
BPF_W (32-bit) and BPF_DW (64-bit) size modifiers, load-acquires and
store-releases also support BPF_B (8-bit) and BPF_H (16-bit). An 8- or
16-bit load-acquire zero-extends the value before writing it to a 32-bit
register, just like ARM64 instruction LDAPRH and friends.
As an example (assuming little-endian):
long foo(long *ptr) {
return __atomic_load_n(ptr, __ATOMIC_ACQUIRE);
}
foo() can be compiled to:
db 10 00 00 00 01 00 00 r0 = load_acquire((u64 *)(r1 + 0x0))
95 00 00 00 00 00 00 00 exit
opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX
imm (0x00000100): BPF_LOAD_ACQ
Similarly:
void bar(short *ptr, short val) {
__atomic_store_n(ptr, val, __ATOMIC_RELEASE);
}
bar() can be compiled to:
cb 21 00 00 10 01 00 00 store_release((u16 *)(r1 + 0x0), w2)
95 00 00 00 00 00 00 00 exit
opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX
imm (0x00000110): BPF_STORE_REL
Inline assembly is also supported.
Add a pre-defined macro, __BPF_FEATURE_LOAD_ACQ_STORE_REL, to let
developers detect this new feature. It can also be disabled using a new
llc option, -disable-load-acq-store-rel.
Using __ATOMIC_RELAXED for __atomic_store{,_n}() will generate a "plain"
store (BPF_MEM | BPF_STX) instruction:
void foo(short *ptr, short val) {
__atomic_store_n(ptr, val, __ATOMIC_RELAXED);
}
6b 21 00 00 00 00 00 00 *(u16 *)(r1 + 0x0) = w2
95 00 00 00 00 00 00 00 exit
Similarly, using __ATOMIC_RELAXED for __atomic_load{,_n}() will generate
a zero-extending, "plain" load (BPF_MEM | BPF_LDX) instruction:
int foo(char *ptr) {
return __atomic_load_n(ptr, __ATOMIC_RELAXED);
}
71 11 00 00 00 00 00 00 w1 = *(u8 *)(r1 + 0x0)
bc 10 08 00 00 00 00 00 w0 = (s8)w1
95 00 00 00 00 00 00 00 exit
Currently __ATOMIC_CONSUME is an alias for __ATOMIC_ACQUIRE. Using
__ATOMIC_SEQ_CST ("sequentially consistent") is not supported yet and
will cause an error:
$ clang --target=bpf -mcpu=v4 -c bar.c > /dev/null
bar.c:1:5: error: sequentially consistent (seq_cst) atomic load/store is
not supported
1 | int foo(int *ptr) { return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); }
| ^
...
Finally, rename those isST*() and isLD*() helper functions in
BPFMISimplifyPatchable.cpp based on what the instructions actually do,
rather than their instruction class.
[1]
https://lore.kernel.org/all/20240729183246.4110549-1-yepeilin@google.com/
135 lines
3.2 KiB
TableGen
135 lines
3.2 KiB
TableGen
//===-- BPFInstrFormats.td - BPF Instruction Formats -------*- tablegen -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
class BPFOpClass<bits<3> val> {
|
|
bits<3> Value = val;
|
|
}
|
|
|
|
def BPF_LD : BPFOpClass<0x0>;
|
|
def BPF_LDX : BPFOpClass<0x1>;
|
|
def BPF_ST : BPFOpClass<0x2>;
|
|
def BPF_STX : BPFOpClass<0x3>;
|
|
def BPF_ALU : BPFOpClass<0x4>;
|
|
def BPF_JMP : BPFOpClass<0x5>;
|
|
def BPF_JMP32 : BPFOpClass<0x6>;
|
|
def BPF_ALU64 : BPFOpClass<0x7>;
|
|
|
|
class BPFSrcType<bits<1> val> {
|
|
bits<1> Value = val;
|
|
}
|
|
|
|
def BPF_K : BPFSrcType<0x0>;
|
|
def BPF_X : BPFSrcType<0x1>;
|
|
|
|
class BPFArithOp<bits<4> val> {
|
|
bits<4> Value = val;
|
|
}
|
|
|
|
def BPF_ADD : BPFArithOp<0x0>;
|
|
def BPF_SUB : BPFArithOp<0x1>;
|
|
def BPF_MUL : BPFArithOp<0x2>;
|
|
def BPF_DIV : BPFArithOp<0x3>;
|
|
def BPF_OR : BPFArithOp<0x4>;
|
|
def BPF_AND : BPFArithOp<0x5>;
|
|
def BPF_LSH : BPFArithOp<0x6>;
|
|
def BPF_RSH : BPFArithOp<0x7>;
|
|
def BPF_NEG : BPFArithOp<0x8>;
|
|
def BPF_MOD : BPFArithOp<0x9>;
|
|
def BPF_XOR : BPFArithOp<0xa>;
|
|
def BPF_MOV : BPFArithOp<0xb>;
|
|
def BPF_ARSH : BPFArithOp<0xc>;
|
|
def BPF_END : BPFArithOp<0xd>;
|
|
|
|
def BPF_XCHG : BPFArithOp<0xe>;
|
|
def BPF_CMPXCHG : BPFArithOp<0xf>;
|
|
|
|
class BPFAtomicOp<bits<5> val> {
|
|
bits<5> Value = val;
|
|
}
|
|
|
|
def BPF_LOAD_ACQ : BPFAtomicOp<0x10>;
|
|
def BPF_STORE_REL : BPFAtomicOp<0x11>;
|
|
|
|
class BPFEndDir<bits<1> val> {
|
|
bits<1> Value = val;
|
|
}
|
|
|
|
def BPF_TO_LE : BPFSrcType<0x0>;
|
|
def BPF_TO_BE : BPFSrcType<0x1>;
|
|
|
|
class BPFJumpOp<bits<4> val> {
|
|
bits<4> Value = val;
|
|
}
|
|
|
|
def BPF_JA : BPFJumpOp<0x0>;
|
|
def BPF_JEQ : BPFJumpOp<0x1>;
|
|
def BPF_JGT : BPFJumpOp<0x2>;
|
|
def BPF_JGE : BPFJumpOp<0x3>;
|
|
def BPF_JSET : BPFJumpOp<0x4>;
|
|
def BPF_JNE : BPFJumpOp<0x5>;
|
|
def BPF_JSGT : BPFJumpOp<0x6>;
|
|
def BPF_JSGE : BPFJumpOp<0x7>;
|
|
def BPF_CALL : BPFJumpOp<0x8>;
|
|
def BPF_EXIT : BPFJumpOp<0x9>;
|
|
def BPF_JLT : BPFJumpOp<0xa>;
|
|
def BPF_JLE : BPFJumpOp<0xb>;
|
|
def BPF_JSLT : BPFJumpOp<0xc>;
|
|
def BPF_JSLE : BPFJumpOp<0xd>;
|
|
def BPF_JCOND : BPFJumpOp<0xe>;
|
|
|
|
class BPFWidthModifer<bits<2> val> {
|
|
bits<2> Value = val;
|
|
}
|
|
|
|
def BPF_W : BPFWidthModifer<0x0>;
|
|
def BPF_H : BPFWidthModifer<0x1>;
|
|
def BPF_B : BPFWidthModifer<0x2>;
|
|
def BPF_DW : BPFWidthModifer<0x3>;
|
|
|
|
class BPFModeModifer<bits<3> val> {
|
|
bits<3> Value = val;
|
|
}
|
|
|
|
def BPF_IMM : BPFModeModifer<0x0>;
|
|
def BPF_ABS : BPFModeModifer<0x1>;
|
|
def BPF_IND : BPFModeModifer<0x2>;
|
|
def BPF_MEM : BPFModeModifer<0x3>;
|
|
def BPF_MEMSX : BPFModeModifer<0x4>;
|
|
def BPF_ATOMIC : BPFModeModifer<0x6>;
|
|
|
|
class BPFAtomicFlag<bits<4> val> {
|
|
bits<4> Value = val;
|
|
}
|
|
|
|
def BPF_FETCH : BPFAtomicFlag<0x1>;
|
|
|
|
class InstBPF<dag outs, dag ins, string asmstr, list<dag> pattern>
|
|
: Instruction {
|
|
field bits<64> Inst;
|
|
field bits<64> SoftFail = 0;
|
|
let Size = 8;
|
|
|
|
let Namespace = "BPF";
|
|
let DecoderNamespace = "BPF";
|
|
|
|
BPFOpClass BPFClass;
|
|
let Inst{58-56} = BPFClass.Value;
|
|
|
|
dag OutOperandList = outs;
|
|
dag InOperandList = ins;
|
|
let AsmString = asmstr;
|
|
let Pattern = pattern;
|
|
}
|
|
|
|
// Pseudo instructions
|
|
class Pseudo<dag outs, dag ins, string asmstr, list<dag> pattern>
|
|
: InstBPF<outs, ins, asmstr, pattern> {
|
|
let Inst{63-0} = 0;
|
|
let isPseudo = 1;
|
|
}
|