shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reapply "MCFragment: Use trailing data for fixed-size part" (#150846)

Browse Source 
The fixed-size content of the MCFragment object is now stored as
trailing data, replacing ContentStart/ContentEnd with ContentSize. The
available space for trailing data is tracked using `FragSpace`. If the
available space is insufficient, a new block is allocated within the
bump allocator `MCObjectStreamer::FragStorage`.

FragList::Tail cannot be reused when switching sections or subsections,
as it is not associated with the fragment space tracked by `FragSpace`.
Instead, allocate a new fragment, which becomes less expensive after #150574.

Data can only be appended to the tail fragment of a subsection, not to
fragments in the middle. Post-assembler-layout adjustments (such as
.llvm_addrsig and .llvm.call-graph-profile) have been updated to use the
variable-size part instead.

---

This reverts commit a2fef664c29a53bfa8a66927fcf8b2e5c9da4876,
which reverted the innocent f1aa6050bd90f8ec4273da55d362e23905ad3a81 .
Commit df71243fa885cd3db701dc35a0c8d157adaf93b3, the MCOrgFragment fix,
has fixed the root cause of https://github.com/ClangBuiltLinux/linux/issues/2116
This commit is contained in:
Fangrui Song 2025-08-04 09:10:42 -07:00
parent b738f630f7
commit e8fc808bf8
7 changed files with 142 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
llvm/include/llvm/MC/MCObjectStreamer.h
View File

@ -52,6 +52,9 @@ class MCObjectStreamer : public MCStreamer {
DenseMap<const MCSymbol *, SmallVector<PendingAssignment, 1>>
pendingAssignments;
SmallVector<std::unique_ptr<uint8_t[]>, 0> FragStorage;
// Available bytes in the current block for trailing data or new fragments.
size_t FragSpace = 0;
// Used to allocate special fragments that do not use MCFragment's fixed-size
// part.
BumpPtrAllocator SpecialFragAllocator;
@ -86,25 +89,28 @@ public:
/// \name MCStreamer Interface
/// @{
uint8_t *getCurFragEnd() const {
return reinterpret_cast<uint8_t *>(CurFrag + 1) + CurFrag->getFixedSize();
}
MCFragment *allocFragSpace(size_t Headroom);
// Add a new fragment to the current section without a variable-size tail.
void newFragment();
template <typename FT, typename... Args> FT *allocFragment(Args &&...args) {
auto *F = new (SpecialFragAllocator.Allocate(sizeof(FT), alignof(FT)))
FT(std::forward<Args>(args)...);
return F;
}
// Add a new special fragment to the current section and start a new empty
// fragment.
template <typename FT, typename... Args>
FT *newSpecialFragment(Args &&...args) {
auto *F = allocFragment<FT, Args...>(std::forward<Args>(args)...);
auto *F = new (SpecialFragAllocator.Allocate(sizeof(FT), alignof(FT)))
FT(std::forward<Args>(args)...);
addSpecialFragment(F);
return F;
}
void ensureHeadroom(size_t Headroom);
void appendContents(ArrayRef<char> Contents);
void appendContents(size_t Num, char Elt);
void appendContents(size_t Num, uint8_t Elt);
// Add a fixup to the current fragment. Call ensureHeadroom beforehand to
// ensure the fixup and appended content apply to the same fragment.
void addFixup(const MCExpr *Value, MCFixupKind Kind);
void emitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override;

45
llvm/include/llvm/MC/MCSection.h
View File

@ -93,10 +93,10 @@ private:
bool AllowAutoPadding : 1;
// Track content and fixups for the fixed-size part as fragments are
// appended to the section. The content remains immutable, except when
// modified by applyFixup.
uint32_t ContentStart = 0;
uint32_t ContentEnd = 0;
// appended to the section. The content is stored as trailing data of the
// MCFragment. The content remains immutable, except when modified by
// applyFixup.
uint32_t FixedSize = 0;
uint32_t FixupStart = 0;
uint32_t FixupEnd = 0;
@ -189,18 +189,6 @@ public:
//== Content-related functions manage parent's storage using ContentStart and
// ContentSize.
// Get a SmallVector reference. The caller should call doneAppending to update
// `ContentEnd`.
SmallVectorImpl<char> &getContentsForAppending();
void doneAppending();
void appendContents(ArrayRef<char> Contents) {
getContentsForAppending().append(Contents.begin(), Contents.end());
doneAppending();
}
void appendContents(size_t Num, char Elt) {
getContentsForAppending().append(Num, Elt);
doneAppending();
}
MutableArrayRef<char> getContents();
ArrayRef<char> getContents() const;
@ -209,10 +197,10 @@ public:
MutableArrayRef<char> getVarContents();
ArrayRef<char> getVarContents() const;
size_t getFixedSize() const { return ContentEnd - ContentStart; }
size_t getFixedSize() const { return FixedSize; }
size_t getVarSize() const { return VarContentEnd - VarContentStart; }
size_t getSize() const {
return ContentEnd - ContentStart + (VarContentEnd - VarContentStart);
return FixedSize + (VarContentEnd - VarContentStart);
}
//== Fixup-related functions manage parent's storage using FixupStart and
@ -630,28 +618,11 @@ public:
bool isBssSection() const { return IsBss; }
};
inline SmallVectorImpl<char> &MCFragment::getContentsForAppending() {
SmallVectorImpl<char> &S = getParent()->ContentStorage;
if (LLVM_UNLIKELY(ContentEnd != S.size())) {
// Move the elements to the end. Reserve space to avoid invalidating
// S.begin()+I for `append`.
auto Size = ContentEnd - ContentStart;
auto I = std::exchange(ContentStart, S.size());
S.reserve(S.size() + Size);
S.append(S.begin() + I, S.begin() + I + Size);
}
return S;
}
inline void MCFragment::doneAppending() {
ContentEnd = getParent()->ContentStorage.size();
}
inline MutableArrayRef<char> MCFragment::getContents() {
return MutableArrayRef(getParent()->ContentStorage)
.slice(ContentStart, ContentEnd - ContentStart);
return {reinterpret_cast<char *>(this + 1), FixedSize};
}
inline ArrayRef<char> MCFragment::getContents() const {
return ArrayRef(getParent()->ContentStorage)
.slice(ContentStart, ContentEnd - ContentStart);
return {reinterpret_cast<const char *>(this + 1), FixedSize};
}
inline MutableArrayRef<char> MCFragment::getVarContents() {

115
llvm/lib/MC/MCObjectStreamer.cpp
View File

@ -46,23 +46,79 @@ MCAssembler *MCObjectStreamer::getAssemblerPtr() {
return nullptr;
}
constexpr size_t FragBlockSize = 16384;
// Ensure the new fragment can at least store a few bytes.
constexpr size_t NewFragHeadroom = 8;
static_assert(NewFragHeadroom >= alignof(MCFragment));
static_assert(FragBlockSize >= sizeof(MCFragment) + NewFragHeadroom);
MCFragment *MCObjectStreamer::allocFragSpace(size_t Headroom) {
auto Size = std::max(FragBlockSize, sizeof(MCFragment) + Headroom);
FragSpace = Size - sizeof(MCFragment);
auto Block = std::unique_ptr<uint8_t[]>(new uint8_t[Size]);
auto *F = reinterpret_cast<MCFragment *>(Block.get());
FragStorage.push_back(std::move(Block));
return F;
}
void MCObjectStreamer::newFragment() {
addFragment(allocFragment<MCFragment>());
MCFragment *F;
if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
auto End = reinterpret_cast<size_t>(getCurFragEnd());
F = reinterpret_cast<MCFragment *>(
alignToPowerOf2(End, alignof(MCFragment)));
FragSpace -= size_t(F) - End + sizeof(MCFragment);
} else {
F = allocFragSpace(0);
}
new (F) MCFragment();
addFragment(F);
}
void MCObjectStreamer::ensureHeadroom(size_t Headroom) {
if (Headroom <= FragSpace)
return;
auto *F = allocFragSpace(Headroom);
new (F) MCFragment();
addFragment(F);
}
void MCObjectStreamer::addSpecialFragment(MCFragment *Frag) {
assert(Frag->getKind() != MCFragment::FT_Data &&
"Frag should have a variable-size tail");
// Frag is not connected to FragSpace. Before modifying CurFrag with
// addFragment(Frag), allocate an empty fragment to maintain FragSpace
// connectivity, potentially reusing CurFrag's associated space.
MCFragment *F;
if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
auto End = reinterpret_cast<size_t>(getCurFragEnd());
F = reinterpret_cast<MCFragment *>(
alignToPowerOf2(End, alignof(MCFragment)));
FragSpace -= size_t(F) - End + sizeof(MCFragment);
} else {
F = allocFragSpace(0);
}
new (F) MCFragment();
addFragment(Frag);
newFragment();
addFragment(F);
}
void MCObjectStreamer::appendContents(ArrayRef<char> Contents) {
CurFrag->appendContents(Contents);
ensureHeadroom(Contents.size());
assert(FragSpace >= Contents.size());
llvm::copy(Contents, getCurFragEnd());
CurFrag->FixedSize += Contents.size();
FragSpace -= Contents.size();
}
void MCObjectStreamer::appendContents(size_t Num, char Elt) {
CurFrag->appendContents(Num, Elt);
void MCObjectStreamer::appendContents(size_t Num, uint8_t Elt) {
ensureHeadroom(Num);
MutableArrayRef<uint8_t> Data(getCurFragEnd(), Num);
llvm::fill(Data, Elt);
CurFrag->FixedSize += Num;
FragSpace -= Num;
}
void MCObjectStreamer::addFixup(const MCExpr *Value, MCFixupKind Kind) {
@ -115,6 +171,8 @@ void MCObjectStreamer::reset() {
}
EmitEHFrame = true;
EmitDebugFrame = false;
FragStorage.clear();
FragSpace = 0;
SpecialFragAllocator.Reset();
MCStreamer::reset();
}
@ -144,7 +202,6 @@ void MCObjectStreamer::emitCFISections(bool EH, bool Debug, bool SFrame) {
void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
SMLoc Loc) {
MCStreamer::emitValueImpl(Value, Size, Loc);
MCFragment *DF = getCurrentFragment();
MCDwarfLineEntry::make(this, getCurrentSectionOnly());
@ -159,9 +216,9 @@ void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
emitIntValue(AbsValue, Size);
return;
}
DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
MCFixup::getDataKindForSize(Size)));
DF->appendContents(Size, 0);
ensureHeadroom(Size);
addFixup(Value, MCFixup::getDataKindForSize(Size));
appendContents(Size, 0);
}
MCSymbol *MCObjectStreamer::emitCFILabel() {
@ -195,7 +252,7 @@ void MCObjectStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) {
// section.
MCFragment *F = CurFrag;
Symbol->setFragment(F);
Symbol->setOffset(F->getContents().size());
Symbol->setOffset(F->getFixedSize());
emitPendingAssignments(Symbol);
}
@ -261,6 +318,21 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
F0 = CurFrag;
}
// To maintain connectivity between CurFrag and FragSpace when CurFrag is
// modified, allocate an empty fragment and append it to the fragment list.
// (Subsections[I].second.Tail is not connected to FragSpace.)
MCFragment *F;
if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
auto End = reinterpret_cast<size_t>(getCurFragEnd());
F = reinterpret_cast<MCFragment *>(
alignToPowerOf2(End, alignof(MCFragment)));
FragSpace -= size_t(F) - End + sizeof(MCFragment);
} else {
F = allocFragSpace(0);
}
new (F) MCFragment();
F->setParent(Section);
auto &Subsections = Section->Subsections;
size_t I = 0, E = Subsections.size();
while (I != E && Subsections[I].first < Subsection)
@ -268,13 +340,16 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
// If the subsection number is not in the sorted Subsections list, create a
// new fragment list.
if (I == E || Subsections[I].first != Subsection) {
auto *F = allocFragment<MCFragment>();
F->setParent(Section);
Subsections.insert(Subsections.begin() + I,
{Subsection, MCSection::FragList{F, F}});
Section->CurFragList = &Subsections[I].second;
CurFrag = F;
} else {
Section->CurFragList = &Subsections[I].second;
CurFrag = Subsections[I].second.Tail;
// Ensure CurFrag is associated with FragSpace.
addFragment(F);
}
Section->CurFragList = &Subsections[I].second;
CurFrag = Section->CurFragList->Tail;
// Define the section symbol at subsection 0's initial fragment if required.
if (!NewSec)
@ -345,11 +420,15 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst,
MCFragment *F = getCurrentFragment();
// Append the instruction to the data fragment.
size_t CodeOffset = F->getContents().size();
size_t CodeOffset = getCurFragSize();
SmallString<16> Content;
SmallVector<MCFixup, 1> Fixups;
getAssembler().getEmitter().encodeInstruction(
Inst, F->getContentsForAppending(), Fixups, STI);
F->doneAppending();
getAssembler().getEmitter().encodeInstruction(Inst, Content, Fixups, STI);
appendContents(Content);
if (CurFrag != F) {
F = CurFrag;
CodeOffset = 0;
}
F->setHasInstructions(STI);
if (Fixups.empty())

3
llvm/lib/MC/MCWin64EH.cpp
View File

@ -318,6 +318,9 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
// Emit the epilog instructions.
if (EnableUnwindV2) {
// Ensure the fixups and appended content apply to the same fragment.
OS->ensureHeadroom(info->EpilogMap.size() * 2);
bool IsLast = true;
for (const auto &Epilog : llvm::reverse(info->EpilogMap)) {
if (IsLast) {

5
llvm/lib/MC/MCWinCOFFStreamer.cpp
View File

@ -279,6 +279,7 @@ void MCWinCOFFStreamer::emitCOFFSymbolIndex(MCSymbol const *Symbol) {
void MCWinCOFFStreamer::emitCOFFSectionIndex(const MCSymbol *Symbol) {
visitUsedSymbol(*Symbol);
const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(Symbol, getContext());
ensureHeadroom(2);
addFixup(SRE, FK_SecRel_2);
appendContents(2, 0);
}
@ -292,6 +293,7 @@ void MCWinCOFFStreamer::emitCOFFSecRel32(const MCSymbol *Symbol,
if (Offset)
MCE = MCBinaryExpr::createAdd(
MCE, MCConstantExpr::create(Offset, getContext()), getContext());
ensureHeadroom(4);
addFixup(MCE, FK_SecRel_4);
// Emit 4 bytes (zeros) to the object file.
appendContents(4, 0);
@ -307,6 +309,7 @@ void MCWinCOFFStreamer::emitCOFFImgRel32(const MCSymbol *Symbol,
if (Offset)
MCE = MCBinaryExpr::createAdd(
MCE, MCConstantExpr::create(Offset, getContext()), getContext());
ensureHeadroom(4);
addFixup(MCE, FK_Data_4);
// Emit 4 bytes (zeros) to the object file.
appendContents(4, 0);
@ -317,6 +320,7 @@ void MCWinCOFFStreamer::emitCOFFSecNumber(MCSymbol const *Symbol) {
// Create Symbol for section number.
const MCExpr *MCE = MCCOFFSectionNumberTargetExpr::create(
*Symbol, this->getWriter(), getContext());
ensureHeadroom(4);
addFixup(MCE, FK_Data_4);
// Emit 4 bytes (zeros) to the object file.
appendContents(4, 0);
@ -327,6 +331,7 @@ void MCWinCOFFStreamer::emitCOFFSecOffset(MCSymbol const *Symbol) {
// Create Symbol for section offset.
const MCExpr *MCE =
MCCOFFSectionOffsetTargetExpr::create(*Symbol, getContext());
ensureHeadroom(4);
addFixup(MCE, FK_Data_4);
// Emit 4 bytes (zeros) to the object file.
appendContents(4, 0);

6
llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
View File

@ -1034,12 +1034,14 @@ MCELFStreamer &MipsTargetELFStreamer::getStreamer() {
void MipsTargetELFStreamer::emitGPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_GPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
S.ensureHeadroom(8);
// fixup_Mips_GPREL32 desginates R_MIPS_GPREL32+R_MIPS_64 on MIPS64.
S.addFixup(Value, Mips::fixup_Mips_GPREL32);
S.appendContents(8, 0);
@ -1047,24 +1049,28 @@ void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) {
void MipsTargetELFStreamer::emitDTPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_DTPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitDTPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
S.ensureHeadroom(8);
S.addFixup(Value, Mips::fixup_Mips_DTPREL64);
S.appendContents(8, 0);
}
void MipsTargetELFStreamer::emitTPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_TPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitTPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
S.ensureHeadroom(8);
S.addFixup(Value, Mips::fixup_Mips_TPREL64);
S.appendContents(8, 0);
}

10
llvm/test/MC/ELF/many-instructions.s Normal file
View File

@ -0,0 +1,10 @@
# REQUIRES: asserts
# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o /dev/null -debug-only=mc-dump 2>&1 | grep -E -o '[0-9]+ Data Size:[0-9]+' | FileCheck %s
## Test that encodeInstruction may cause a new fragment to be created.
# CHECK: 0 Data Size:16220
# CHECK: 16220 Data Size:160
.rept 16384/10
movabsq $foo, %rax
.endr