shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[BOLT] Encode BAT using ULEB128 (#76899)

Browse Source 
Reduces BAT section size, bytes:
- large binary: 38676872 -> 23262524 (0.60x),
- medium binary (trunk clang): 5938004 -> 3213504 (0.54x),
- small binary (X86/bolt-address-translation.test): 1436 -> 680 (0.47x).

Test Plan: Updated bolt/test/X86/bolt-address-translation.test
This commit is contained in:
Amir Ayupov 2023-12-21 12:16:59 -08:00
parent a7cf0a1f7f
commit 565f40d66b
4 changed files with 40 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
bolt/docs/BAT.md
View File

@ -59,24 +59,24 @@ Fragment linkage header
### Functions table
Header:
| Entry | Width | Description |
| Entry | Encoding | Description |
| ------ | ----- | ----------- |
| `NumFuncs` | 4B | Number of functions in the functions table |
| `NumFuncs` | ULEB128 | Number of functions in the functions table |
The header is followed by Functions table with `NumFuncs` entries.
| Entry | Width | Description |
| Entry | Encoding | Description |
| ------ | ------| ----------- |
| `Address` | 8B | Function address in the output binary |
| `NumEntries` | 4B | Number of address translation entries for a function |
| `Address` | ULEB128 | Function address in the output binary |
| `NumEntries` | ULEB128 | Number of address translation entries for a function |
Function header is followed by `NumEntries` pairs of offsets for current
function.
### Address translation table
| Entry | Width | Description |
| Entry | Encoding | Description |
| ------ | ------| ----------- |
| `OutputAddr` | 4B | Function offset in output binary |
| `InputAddr` | 4B | Function offset in input binary with `BRANCHENTRY` top bit |
| `OutputAddr` | ULEB128 | Function offset in output binary |
| `InputAddr` | ULEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit |
`BRANCHENTRY` bit denotes whether a given offset pair is a control flow source
(branch or call instruction). If not set, it signifies a control flow target
@ -86,12 +86,12 @@ function.
Following Functions table, fragment linkage table is encoded to link split
cold fragments with main (hot) fragment.
Header:
| Entry | Width | Description |
| Entry | Encoding | Description |
| ------ | ------------ | ----------- |
| `NumColdEntries` | 4B | Number of split functions in the functions table |
| `NumColdEntries` | ULEB128 | Number of split functions in the functions table |
`NumColdEntries` pairs of addresses follow:
| Entry | Width | Description |
| Entry | Encoding | Description |
| ------ | ------| ----------- |
| `ColdAddress` | 8B | Cold fragment address in output binary |
| `HotAddress` | 8B | Hot fragment address in output binary |
| `ColdAddress` | ULEB128 | Cold fragment address in output binary |
| `HotAddress` | ULEB128 | Hot fragment address in output binary |

2
bolt/include/bolt/Profile/BoltAddressTranslation.h
View File

@ -125,7 +125,7 @@ private:
/// Identifies the address of a control-flow changing instructions in a
/// translation map entry
const static uint32_t BRANCHENTRY = 0x80000000;
const static uint32_t BRANCHENTRY = 0x1;
};
} // namespace bolt

60
bolt/lib/Profile/BoltAddressTranslation.cpp
View File

@ -10,6 +10,8 @@
#include "bolt/Core/BinaryFunction.h"
#include "llvm/Support/DataExtractor.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/LEB128.h"
#define DEBUG_TYPE "bolt-bat"
@ -44,7 +46,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map,
// and this deleted block will both share the same output address (the same
// key), and we need to map back. We choose here to privilege the successor by
// allowing it to overwrite the previously inserted key in the map.
Map[BBOutputOffset] = BBInputOffset;
Map[BBOutputOffset] = BBInputOffset << 1;
const auto &IOAddressMap =
BB.getFunction()->getBinaryContext().getIOAddressMap();
@ -61,8 +63,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map,
LLVM_DEBUG(dbgs() << " Key: " << Twine::utohexstr(OutputOffset) << " Val: "
<< Twine::utohexstr(InputOffset) << " (branch)\n");
Map.insert(
std::pair<uint32_t, uint32_t>(OutputOffset, InputOffset | BRANCHENTRY));
Map.insert(std::pair<uint32_t, uint32_t>(OutputOffset,
(InputOffset << 1) | BRANCHENTRY));
}
}
@ -102,7 +104,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
}
const uint32_t NumFuncs = Maps.size();
OS.write(reinterpret_cast<const char *>(&NumFuncs), 4);
encodeULEB128(NumFuncs, OS);
LLVM_DEBUG(dbgs() << "Writing " << NumFuncs << " functions for BAT.\n");
for (auto &MapEntry : Maps) {
const uint64_t Address = MapEntry.first;
@ -110,20 +112,20 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
const uint32_t NumEntries = Map.size();
LLVM_DEBUG(dbgs() << "Writing " << NumEntries << " entries for 0x"
<< Twine::utohexstr(Address) << ".\n");
OS.write(reinterpret_cast<const char *>(&Address), 8);
OS.write(reinterpret_cast<const char *>(&NumEntries), 4);
encodeULEB128(Address, OS);
encodeULEB128(NumEntries, OS);
for (std::pair<const uint32_t, uint32_t> &KeyVal : Map) {
OS.write(reinterpret_cast<const char *>(&KeyVal.first), 4);
OS.write(reinterpret_cast<const char *>(&KeyVal.second), 4);
encodeULEB128(KeyVal.first, OS);
encodeULEB128(KeyVal.second, OS);
}
}
const uint32_t NumColdEntries = ColdPartSource.size();
LLVM_DEBUG(dbgs() << "Writing " << NumColdEntries
<< " cold part mappings.\n");
OS.write(reinterpret_cast<const char *>(&NumColdEntries), 4);
encodeULEB128(NumColdEntries, OS);
for (std::pair<const uint64_t, uint64_t> &ColdEntry : ColdPartSource) {
OS.write(reinterpret_cast<const char *>(&ColdEntry.first), 8);
OS.write(reinterpret_cast<const char *>(&ColdEntry.second), 8);
encodeULEB128(ColdEntry.first, OS);
encodeULEB128(ColdEntry.second, OS);
LLVM_DEBUG(dbgs() << " " << Twine::utohexstr(ColdEntry.first) << " -> "
<< Twine::utohexstr(ColdEntry.second) << "\n");
}
@ -152,26 +154,19 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) {
if (Name.substr(0, 4) != "BOLT")
return make_error_code(llvm::errc::io_error);
if (Buf.size() - Offset < 4)
return make_error_code(llvm::errc::io_error);
const uint32_t NumFunctions = DE.getU32(&Offset);
Error Err(Error::success());
const uint32_t NumFunctions = DE.getULEB128(&Offset, &Err);
LLVM_DEBUG(dbgs() << "Parsing " << NumFunctions << " functions\n");
for (uint32_t I = 0; I < NumFunctions; ++I) {
if (Buf.size() - Offset < 12)
return make_error_code(llvm::errc::io_error);
const uint64_t Address = DE.getU64(&Offset);
const uint32_t NumEntries = DE.getU32(&Offset);
const uint64_t Address = DE.getULEB128(&Offset, &Err);
const uint32_t NumEntries = DE.getULEB128(&Offset, &Err);
MapTy Map;
LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x"
<< Twine::utohexstr(Address) << "\n");
if (Buf.size() - Offset < 8 * NumEntries)
return make_error_code(llvm::errc::io_error);
for (uint32_t J = 0; J < NumEntries; ++J) {
const uint32_t OutputAddr = DE.getU32(&Offset);
const uint32_t InputAddr = DE.getU32(&Offset);
const uint32_t OutputAddr = DE.getULEB128(&Offset, &Err);
const uint32_t InputAddr = DE.getULEB128(&Offset, &Err);
Map.insert(std::pair<uint32_t, uint32_t>(OutputAddr, InputAddr));
LLVM_DEBUG(dbgs() << Twine::utohexstr(OutputAddr) << " -> "
<< Twine::utohexstr(InputAddr) << "\n");
@ -179,16 +174,11 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) {
Maps.insert(std::pair<uint64_t, MapTy>(Address, Map));
}
if (Buf.size() - Offset < 4)
return make_error_code(llvm::errc::io_error);
const uint32_t NumColdEntries = DE.getU32(&Offset);
const uint32_t NumColdEntries = DE.getULEB128(&Offset, &Err);
LLVM_DEBUG(dbgs() << "Parsing " << NumColdEntries << " cold part mappings\n");
for (uint32_t I = 0; I < NumColdEntries; ++I) {
if (Buf.size() - Offset < 16)
return make_error_code(llvm::errc::io_error);
const uint32_t ColdAddress = DE.getU64(&Offset);
const uint32_t HotAddress = DE.getU64(&Offset);
const uint32_t ColdAddress = DE.getULEB128(&Offset, &Err);
const uint32_t HotAddress = DE.getULEB128(&Offset, &Err);
ColdPartSource.insert(
std::pair<uint64_t, uint64_t>(ColdAddress, HotAddress));
LLVM_DEBUG(dbgs() << Twine::utohexstr(ColdAddress) << " -> "
@ -198,7 +188,7 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) {
outs() << "BOLT-INFO: Parsed " << NumColdEntries
<< " BAT cold-to-hot entries\n";
return std::error_code();
return errorToErrorCode(std::move(Err));
}
void BoltAddressTranslation::dump(raw_ostream &OS) {
@ -209,7 +199,7 @@ void BoltAddressTranslation::dump(raw_ostream &OS) {
OS << "BB mappings:\n";
for (const auto &Entry : MapEntry.second) {
const bool IsBranch = Entry.second & BRANCHENTRY;
const uint32_t Val = Entry.second & ~BRANCHENTRY;
const uint32_t Val = Entry.second >> 1; // dropping BRANCHENTRY bit
OS << "0x" << Twine::utohexstr(Entry.first) << " -> "
<< "0x" << Twine::utohexstr(Val);
if (IsBranch)
@ -244,7 +234,7 @@ uint64_t BoltAddressTranslation::translate(uint64_t FuncAddress,
--KeyVal;
const uint32_t Val = KeyVal->second & ~BRANCHENTRY;
const uint32_t Val = KeyVal->second >> 1; // dropping BRANCHENTRY bit
// Branch source addresses are translated to the first instruction of the
// source BB to avoid accounting for modifications BOLT may have made in the
// BB regarding deletion/addition of instructions.

2
bolt/test/X86/bolt-address-translation.test
View File

@ -37,7 +37,7 @@
# CHECK: BOLT: 3 out of 7 functions were overwritten.
# CHECK: BOLT-INFO: Wrote 6 BAT maps
# CHECK: BOLT-INFO: Wrote 3 BAT cold-to-hot entries
# CHECK: BOLT-INFO: BAT section size (bytes): 1436
# CHECK: BOLT-INFO: BAT section size (bytes): 680
#
# usqrt mappings (hot part). We match against any key (left side containing
# the bolted binary offsets) because BOLT may change where it puts instructions