410 lines
14 KiB
C++
410 lines
14 KiB
C++
//===---- aarch64.cpp - Generic JITLink aarch64 edge kinds, utilities -----===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Generic utilities for graphs representing aarch64 objects.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/ExecutionEngine/JITLink/aarch64.h"
|
|
|
|
#include "llvm/Support/BinaryStreamWriter.h"
|
|
|
|
#define DEBUG_TYPE "jitlink"
|
|
|
|
namespace llvm {
|
|
namespace jitlink {
|
|
namespace aarch64 {
|
|
|
|
const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00,
|
|
0x00, 0x00, 0x00, 0x00};
|
|
|
|
const char PointerJumpStubContent[12] = {
|
|
0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page21
|
|
0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12]
|
|
0x00, 0x02, 0x1f, (char)0xd6u // BR x16
|
|
};
|
|
|
|
const char ReentryTrampolineContent[8] = {
|
|
(char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]
|
|
0x00, 0x00, 0x00, (char)0x94 // BL
|
|
};
|
|
|
|
const char *getEdgeKindName(Edge::Kind R) {
|
|
switch (R) {
|
|
case Pointer64:
|
|
return "Pointer64";
|
|
case Pointer64Authenticated:
|
|
return "Pointer64Authenticated";
|
|
case Pointer32:
|
|
return "Pointer32";
|
|
case Delta64:
|
|
return "Delta64";
|
|
case Delta32:
|
|
return "Delta32";
|
|
case NegDelta64:
|
|
return "NegDelta64";
|
|
case NegDelta32:
|
|
return "NegDelta32";
|
|
case Branch26PCRel:
|
|
return "Branch26PCRel";
|
|
case MoveWide16:
|
|
return "MoveWide16";
|
|
case LDRLiteral19:
|
|
return "LDRLiteral19";
|
|
case TestAndBranch14PCRel:
|
|
return "TestAndBranch14PCRel";
|
|
case CondBranch19PCRel:
|
|
return "CondBranch19PCRel";
|
|
case ADRLiteral21:
|
|
return "ADRLiteral21";
|
|
case Page21:
|
|
return "Page21";
|
|
case PageOffset12:
|
|
return "PageOffset12";
|
|
case GotPageOffset15:
|
|
return "GotPageOffset15";
|
|
case RequestGOTAndTransformToPage21:
|
|
return "RequestGOTAndTransformToPage21";
|
|
case RequestGOTAndTransformToPageOffset12:
|
|
return "RequestGOTAndTransformToPageOffset12";
|
|
case RequestGOTAndTransformToPageOffset15:
|
|
return "RequestGOTAndTransformToPageOffset15";
|
|
case RequestGOTAndTransformToDelta32:
|
|
return "RequestGOTAndTransformToDelta32";
|
|
case RequestTLVPAndTransformToPage21:
|
|
return "RequestTLVPAndTransformToPage21";
|
|
case RequestTLVPAndTransformToPageOffset12:
|
|
return "RequestTLVPAndTransformToPageOffset12";
|
|
case RequestTLSDescEntryAndTransformToPage21:
|
|
return "RequestTLSDescEntryAndTransformToPage21";
|
|
case RequestTLSDescEntryAndTransformToPageOffset12:
|
|
return "RequestTLSDescEntryAndTransformToPageOffset12";
|
|
default:
|
|
return getGenericEdgeKindName(R);
|
|
}
|
|
}
|
|
|
|
// Write a 64-bit GPR -> GPR move.
|
|
template <typename AppendFtor>
|
|
static Error writeMovRegRegSeq(AppendFtor &Append, uint64_t DstReg,
|
|
uint64_t SrcReg) {
|
|
assert(DstReg < 32 && "Dst reg out of range");
|
|
assert(SrcReg < 32 && "Src reg out of range");
|
|
|
|
if (DstReg == SrcReg)
|
|
return Error::success();
|
|
|
|
constexpr uint32_t MOVGPR64Template = 0xaa0003e0;
|
|
constexpr uint32_t DstRegIndex = 0;
|
|
constexpr uint32_t SrcRegIndex = 16;
|
|
uint32_t Instr = MOVGPR64Template;
|
|
Instr |= DstReg << DstRegIndex;
|
|
Instr |= SrcReg << SrcRegIndex;
|
|
return Append(Instr);
|
|
}
|
|
|
|
// Generate a sequence of imm writes to assign the given value.
|
|
template <typename AppendFtor>
|
|
static Error writeMovRegImm64Seq(AppendFtor &Append, uint64_t Reg,
|
|
uint64_t Imm) {
|
|
assert(Reg < 32 && "Invalid register number");
|
|
|
|
constexpr uint32_t MovRegImm64Template = 0xd2800000;
|
|
constexpr unsigned PreserveBitIndex = 29;
|
|
constexpr unsigned ShiftBitsIndex = 21;
|
|
constexpr unsigned ImmBitsIndex = 5;
|
|
|
|
bool PreserveRegValue = false;
|
|
for (unsigned I = 0; I != 4; ++I) {
|
|
uint32_t ImmBits = Imm & 0xffff;
|
|
Imm >>= 16;
|
|
|
|
// Skip any all-zero immediates after the first one.
|
|
if (PreserveRegValue && !ImmBits)
|
|
continue;
|
|
|
|
uint32_t Instr = MovRegImm64Template;
|
|
Instr |= PreserveRegValue << PreserveBitIndex;
|
|
Instr |= (I << ShiftBitsIndex);
|
|
Instr |= ImmBits << ImmBitsIndex;
|
|
Instr |= Reg;
|
|
if (auto Err = Append(Instr))
|
|
return Err;
|
|
PreserveRegValue = true;
|
|
}
|
|
|
|
return Error::success();
|
|
}
|
|
|
|
template <typename AppendFtor>
|
|
static Error
|
|
writePACSignSeq(AppendFtor &Append, unsigned DstReg, orc::ExecutorAddr RawAddr,
|
|
unsigned RawAddrReg, unsigned DiscriminatorReg, unsigned Key,
|
|
uint64_t EncodedDiscriminator, bool AddressDiversify) {
|
|
assert(DstReg < 32 && "DstReg out of range");
|
|
assert(RawAddrReg < 32 && "AddrReg out of range");
|
|
assert(DiscriminatorReg < 32 && "DiscriminatorReg out of range");
|
|
assert(EncodedDiscriminator < 0x10000 && "EncodedDiscriminator out of range");
|
|
|
|
if (AddressDiversify) {
|
|
// Move the address into the discriminator register.
|
|
if (auto Err = writeMovRegRegSeq(Append, DiscriminatorReg, RawAddrReg))
|
|
return Err;
|
|
// Blend encoded discriminator if there is one.
|
|
if (EncodedDiscriminator) {
|
|
constexpr uint32_t MOVKTemplate = 0xf2e00000;
|
|
constexpr unsigned ImmIndex = 5;
|
|
uint32_t BlendInstr = MOVKTemplate;
|
|
BlendInstr |= EncodedDiscriminator << ImmIndex;
|
|
BlendInstr |= DiscriminatorReg;
|
|
if (auto Err = Append(BlendInstr))
|
|
return Err;
|
|
}
|
|
} else if (EncodedDiscriminator) {
|
|
// Move the encoded discriminator into the discriminator register.
|
|
if (auto Err =
|
|
writeMovRegImm64Seq(Append, DiscriminatorReg, EncodedDiscriminator))
|
|
return Err;
|
|
} else
|
|
DiscriminatorReg = 31; // WZR
|
|
|
|
constexpr uint32_t PACTemplate = 0xdac10000;
|
|
constexpr unsigned ZBitIndex = 13;
|
|
constexpr unsigned KeyIndex = 10;
|
|
constexpr unsigned DiscriminatorRegIndex = 5;
|
|
|
|
uint32_t Instr = PACTemplate;
|
|
Instr |= (DiscriminatorReg == 31) << ZBitIndex;
|
|
Instr |= Key << KeyIndex;
|
|
Instr |= DiscriminatorReg << DiscriminatorRegIndex;
|
|
Instr |= DstReg;
|
|
|
|
return Append(Instr);
|
|
}
|
|
|
|
template <typename AppendFtor>
|
|
static Error writeStoreRegSeq(AppendFtor &Append, unsigned DstLocReg,
|
|
unsigned SrcReg) {
|
|
assert(DstLocReg < 32 && "DstLocReg out of range");
|
|
assert(SrcReg < 32 && "SrcReg out of range");
|
|
|
|
constexpr uint32_t STRTemplate = 0xf9000000;
|
|
constexpr unsigned DstLocRegIndex = 5;
|
|
constexpr unsigned SrcRegIndex = 0;
|
|
|
|
uint32_t Instr = STRTemplate;
|
|
Instr |= DstLocReg << DstLocRegIndex;
|
|
Instr |= SrcReg << SrcRegIndex;
|
|
|
|
return Append(Instr);
|
|
}
|
|
|
|
void GOTTableManager::registerExistingEntries() {
|
|
for (auto *EntrySym : GOTSection->symbols()) {
|
|
assert(EntrySym->getBlock().edges_size() == 1 &&
|
|
"GOT block edge count != 1");
|
|
registerPreExistingEntry(EntrySym->getBlock().edges().begin()->getTarget(),
|
|
*EntrySym);
|
|
}
|
|
}
|
|
|
|
void PLTTableManager::registerExistingEntries() {
|
|
for (auto *EntrySym : StubsSection->symbols()) {
|
|
assert(EntrySym->getBlock().edges_size() == 2 &&
|
|
"PLT block edge count != 2");
|
|
auto &GOTSym = EntrySym->getBlock().edges().begin()->getTarget();
|
|
assert(GOTSym.getBlock().edges_size() == 1 && "GOT block edge count != 1");
|
|
registerPreExistingEntry(GOTSym.getBlock().edges().begin()->getTarget(),
|
|
*EntrySym);
|
|
}
|
|
}
|
|
|
|
const char *getPointerSigningFunctionSectionName() { return "$__ptrauth_sign"; }
|
|
|
|
/// Creates a pointer signing function section, block, and symbol to reserve
|
|
/// space for a signing function for this LinkGraph. Clients should insert this
|
|
/// pass in the post-prune phase, and add the paired
|
|
/// lowerPointer64AuthEdgesToSigningFunction pass to the pre-fixup phase.
|
|
Error createEmptyPointerSigningFunction(LinkGraph &G) {
|
|
LLVM_DEBUG({
|
|
dbgs() << "Creating empty pointer signing function for " << G.getName()
|
|
<< "\n";
|
|
});
|
|
|
|
// FIXME: We could put a tighter bound on this if we inspected the ptrauth
|
|
// info encoded in the addend -- the only actually unknown quantity is the
|
|
// fixup location, and we can probably put constraints even on that.
|
|
size_t NumPtrAuthFixupLocations = 0;
|
|
for (auto &Sec : G.sections()) {
|
|
|
|
// No-alloc sections can't have ptrauth edges. We don't need to error out
|
|
// here: applyFixup will catch these edges if any make it to the fixup
|
|
// stage.
|
|
if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)
|
|
continue;
|
|
|
|
for (auto *B : Sec.blocks()) {
|
|
for (auto &E : B->edges())
|
|
NumPtrAuthFixupLocations +=
|
|
E.getKind() == aarch64::Pointer64Authenticated;
|
|
}
|
|
}
|
|
|
|
constexpr size_t MaxPtrSignSeqLength =
|
|
4 + // To materialize the value to sign.
|
|
4 + // To materialize the fixup location.
|
|
3 + // To copy, blend discriminator, and sign
|
|
1; // To store the result.
|
|
|
|
// The maximum number of signing instructions required is the maximum per
|
|
// location, times the number of locations, plus three instructions to
|
|
// materialize the return value and return.
|
|
size_t NumSigningInstrs = NumPtrAuthFixupLocations * MaxPtrSignSeqLength + 3;
|
|
|
|
// Create signing function section.
|
|
auto &SigningSection =
|
|
G.createSection(getPointerSigningFunctionSectionName(),
|
|
orc::MemProt::Read | orc::MemProt::Exec);
|
|
SigningSection.setMemLifetime(orc::MemLifetime::Finalize);
|
|
|
|
size_t SigningFunctionSize = NumSigningInstrs * 4;
|
|
auto &SigningFunctionBlock = G.createMutableContentBlock(
|
|
SigningSection, G.allocateBuffer(SigningFunctionSize),
|
|
orc::ExecutorAddr(), 4, 0);
|
|
G.addAnonymousSymbol(SigningFunctionBlock, 0, SigningFunctionBlock.getSize(),
|
|
true, true);
|
|
|
|
LLVM_DEBUG({
|
|
dbgs() << " " << NumPtrAuthFixupLocations << " location(s) to sign, up to "
|
|
<< NumSigningInstrs << " instructions required ("
|
|
<< formatv("{0:x}", SigningFunctionBlock.getSize()) << " bytes)\n";
|
|
});
|
|
|
|
return Error::success();
|
|
}
|
|
|
|
/// Given a LinkGraph containing Pointer64Auth edges, transform those edges to
|
|
/// Pointer64 and add code to sign the pointers in the executor.
|
|
///
|
|
/// This function will add a $__ptrauth_sign section with finalization-lifetime
|
|
/// containing an anonymous function that will sign all pointers in the graph.
|
|
/// An allocation action will be added to run this function during finalization.
|
|
Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G) {
|
|
LLVM_DEBUG({
|
|
dbgs() << "Writing pointer signing function for " << G.getName() << "\n";
|
|
});
|
|
|
|
constexpr unsigned Reg1 = 8; // Holds pointer value to sign.
|
|
constexpr unsigned Reg2 = 9; // Holds fixup address.
|
|
constexpr unsigned Reg3 = 10; // Temporary for discriminator value if needed.
|
|
|
|
// Find the signing function.
|
|
auto *SigningSection =
|
|
G.findSectionByName(getPointerSigningFunctionSectionName());
|
|
assert(SigningSection && "Siging section missing");
|
|
assert(SigningSection->blocks_size() == 1 &&
|
|
"Unexpected number of blocks in signing section");
|
|
assert(SigningSection->symbols_size() == 1 &&
|
|
"Unexpected number of symbols in signing section");
|
|
|
|
auto &SigningFunctionSym = **SigningSection->symbols().begin();
|
|
auto &SigningFunctionBlock = SigningFunctionSym.getBlock();
|
|
auto SigningFunctionBuf = SigningFunctionBlock.getAlreadyMutableContent();
|
|
|
|
// Write the instructions to the block content.
|
|
BinaryStreamWriter InstrWriter(
|
|
{reinterpret_cast<uint8_t *>(SigningFunctionBuf.data()),
|
|
SigningFunctionBuf.size()},
|
|
G.getEndianness());
|
|
|
|
auto AppendInstr = [&](uint32_t Instr) {
|
|
return InstrWriter.writeInteger(Instr);
|
|
};
|
|
|
|
for (auto &Sec : G.sections()) {
|
|
|
|
if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)
|
|
continue;
|
|
|
|
for (auto *B : Sec.blocks()) {
|
|
for (auto &E : B->edges()) {
|
|
// We're only concerned with Pointer64Authenticated edges here.
|
|
if (E.getKind() != aarch64::Pointer64Authenticated)
|
|
continue;
|
|
|
|
uint64_t EncodedInfo = E.getAddend();
|
|
int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff);
|
|
auto ValueToSign = E.getTarget().getAddress() + RealAddend;
|
|
if (!ValueToSign) {
|
|
LLVM_DEBUG(dbgs() << " " << B->getFixupAddress(E) << " <- null\n");
|
|
E.setAddend(RealAddend);
|
|
E.setKind(aarch64::Pointer64);
|
|
continue;
|
|
}
|
|
|
|
uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff;
|
|
bool AddressDiversify = (EncodedInfo >> 48) & 0x1;
|
|
uint32_t Key = (EncodedInfo >> 49) & 0x3;
|
|
uint32_t HighBits = EncodedInfo >> 51;
|
|
|
|
if (HighBits != 0x1000)
|
|
return make_error<JITLinkError>(
|
|
"Pointer64Auth edge at " +
|
|
formatv("{0:x}", B->getFixupAddress(E).getValue()) +
|
|
" has invalid encoded addend " + formatv("{0:x}", EncodedInfo));
|
|
|
|
LLVM_DEBUG({
|
|
const char *const KeyNames[] = {"IA", "IB", "DA", "DB"};
|
|
dbgs() << " " << B->getFixupAddress(E) << " <- " << ValueToSign
|
|
<< " : key = " << KeyNames[Key] << ", discriminator = "
|
|
<< formatv("{0:x4}", InitialDiscriminator)
|
|
<< ", address diversified = "
|
|
<< (AddressDiversify ? "yes" : "no") << "\n";
|
|
});
|
|
|
|
// Materialize pointer value.
|
|
cantFail(
|
|
writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue()));
|
|
|
|
// Materialize fixup pointer.
|
|
cantFail(writeMovRegImm64Seq(AppendInstr, Reg2,
|
|
B->getFixupAddress(E).getValue()));
|
|
|
|
// Write signing instruction(s).
|
|
cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3,
|
|
Key, InitialDiscriminator, AddressDiversify));
|
|
|
|
// Store signed pointer.
|
|
cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1));
|
|
|
|
// Replace edge with a keep-alive to preserve dependence info.
|
|
E.setKind(Edge::KeepAlive);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Write epilogue. x0 = 0, x1 = 1 is an SPS serialized Error::success value.
|
|
constexpr uint32_t RETInstr = 0xd65f03c0;
|
|
cantFail(writeMovRegImm64Seq(AppendInstr, 0, 0)); // mov x0, #0
|
|
cantFail(writeMovRegImm64Seq(AppendInstr, 1, 1)); // mov x1, #1
|
|
cantFail(AppendInstr(RETInstr)); // ret
|
|
|
|
// Add an allocation action to call the signing function.
|
|
using namespace orc::shared;
|
|
G.allocActions().push_back(
|
|
{cantFail(WrapperFunctionCall::Create<SPSArgList<>>(
|
|
SigningFunctionSym.getAddress())),
|
|
{}});
|
|
|
|
return Error::success();
|
|
}
|
|
|
|
} // namespace aarch64
|
|
} // namespace jitlink
|
|
} // namespace llvm
|