From 07d3a73d70cac6e58ca9002c98e31423c26cc735 Mon Sep 17 00:00:00 2001 From: Kyungwoo Lee Date: Thu, 14 Aug 2025 16:56:54 -0700 Subject: [PATCH] Revert "[CGData] Lazy loading support for stable function map (#151660)" This reverts commit 76dd742f7b32e4d3acf50fab1dbbd897f215837e. --- llvm/include/llvm/CGData/CodeGenData.h | 3 - llvm/include/llvm/CGData/CodeGenData.inc | 2 +- llvm/include/llvm/CGData/StableFunctionMap.h | 65 +------- .../llvm/CGData/StableFunctionMapRecord.h | 48 +----- llvm/lib/CGData/CodeGenData.cpp | 2 +- llvm/lib/CGData/CodeGenDataReader.cpp | 17 +- llvm/lib/CGData/StableFunctionMap.cpp | 70 ++------ llvm/lib/CGData/StableFunctionMapRecord.cpp | 151 +++++------------- llvm/lib/CodeGen/GlobalMergeFunctions.cpp | 10 +- .../ThinLTO/AArch64/cgdata-merge-write.ll | 4 +- llvm/test/tools/llvm-cgdata/empty.test | 4 +- llvm/test/tools/llvm-cgdata/error.test | 4 +- .../merge-combined-funcmap-hashtree.test | 4 +- .../llvm-cgdata/merge-funcmap-archive.test | 8 +- .../llvm-cgdata/merge-funcmap-concat.test | 6 +- .../llvm-cgdata/merge-funcmap-double.test | 7 +- .../llvm-cgdata/merge-funcmap-single.test | 4 +- llvm/tools/llvm-cgdata/Opts.td | 1 - llvm/tools/llvm-cgdata/llvm-cgdata.cpp | 5 - .../CGData/StableFunctionMapTest.cpp | 2 +- 20 files changed, 88 insertions(+), 329 deletions(-) diff --git a/llvm/include/llvm/CGData/CodeGenData.h b/llvm/include/llvm/CGData/CodeGenData.h index e44497a40824..38b96b72ccac 100644 --- a/llvm/include/llvm/CGData/CodeGenData.h +++ b/llvm/include/llvm/CGData/CodeGenData.h @@ -285,9 +285,6 @@ enum CGDataVersion { // Version 3 adds the total size of the Names in the stable function map so // we can skip reading them into the memory for non-assertion builds. Version3 = 3, - // Version 4 adjusts the structure of stable function merging map for - // efficient lazy loading support. - Version4 = 4, CurrentVersion = CG_DATA_INDEX_VERSION }; const uint64_t Version = CGDataVersion::CurrentVersion; diff --git a/llvm/include/llvm/CGData/CodeGenData.inc b/llvm/include/llvm/CGData/CodeGenData.inc index d5fbe2fb9771..94de4c0b017a 100644 --- a/llvm/include/llvm/CGData/CodeGenData.inc +++ b/llvm/include/llvm/CGData/CodeGenData.inc @@ -49,4 +49,4 @@ CG_DATA_SECT_ENTRY(CG_merge, CG_DATA_QUOTE(CG_DATA_MERGE_COMMON), #endif /* Indexed codegen data format version (start from 1). */ -#define CG_DATA_INDEX_VERSION 4 +#define CG_DATA_INDEX_VERSION 3 diff --git a/llvm/include/llvm/CGData/StableFunctionMap.h b/llvm/include/llvm/CGData/StableFunctionMap.h index ea3523c3a329..bcb72e821697 100644 --- a/llvm/include/llvm/CGData/StableFunctionMap.h +++ b/llvm/include/llvm/CGData/StableFunctionMap.h @@ -20,8 +20,6 @@ #include "llvm/ADT/StringMap.h" #include "llvm/IR/StructuralHash.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/MemoryBuffer.h" -#include namespace llvm { @@ -74,37 +72,11 @@ struct StableFunctionMap { IndexOperandHashMap(std::move(IndexOperandHashMap)) {} }; - using StableFunctionEntries = - SmallVector>; - - /// In addition to the deserialized StableFunctionEntry, the struct stores - /// the offsets of corresponding serialized stable function entries, and a - /// once flag for safe lazy loading in a multithreaded environment. - struct EntryStorage { - /// The actual storage of deserialized stable function entries. If the map - /// is lazily loaded, this will be empty until the first access by the - /// corresponding function hash. - StableFunctionEntries Entries; - - private: - /// This is used to deserialize the entry lazily. Each element is the - /// corresponding serialized stable function entry's offset in the memory - /// buffer (StableFunctionMap::Buffer). - /// The offsets are only populated when loading the map lazily, otherwise - /// it is empty. - SmallVector Offsets; - std::once_flag LazyLoadFlag; - friend struct StableFunctionMap; - friend struct StableFunctionMapRecord; - }; - - // Note: DenseMap requires value type to be copyable even if only using - // in-place insertion. Use STL instead. This also affects the - // deletion-while-iteration in finalize(). - using HashFuncsMapType = std::unordered_map; + using HashFuncsMapType = + DenseMap>>; /// Get the HashToFuncs map for serialization. - const HashFuncsMapType &getFunctionMap() const; + const HashFuncsMapType &getFunctionMap() const { return HashToFuncs; } /// Get the NameToId vector for serialization. ArrayRef getNames() const { return IdToName; } @@ -127,19 +99,6 @@ struct StableFunctionMap { /// \returns true if there is no stable function entry. bool empty() const { return size() == 0; } - /// \returns true if there is an entry for the given function hash. - /// This does not trigger lazy loading. - bool contains(HashFuncsMapType::key_type FunctionHash) const { - return HashToFuncs.count(FunctionHash) > 0; - } - - /// \returns the stable function entries for the given function hash. If the - /// map is lazily loaded, it will deserialize the entries if it is not already - /// done, other requests to the same hash at the same time will be blocked - /// until the entries are deserialized. - const StableFunctionEntries & - at(HashFuncsMapType::key_type FunctionHash) const; - enum SizeType { UniqueHashCount, // The number of unique hashes in HashToFuncs. TotalFunctionCount, // The number of total functions in HashToFuncs. @@ -160,31 +119,17 @@ private: /// `StableFunctionEntry` is ready for insertion. void insert(std::unique_ptr FuncEntry) { assert(!Finalized && "Cannot insert after finalization"); - HashToFuncs[FuncEntry->Hash].Entries.emplace_back(std::move(FuncEntry)); + HashToFuncs[FuncEntry->Hash].emplace_back(std::move(FuncEntry)); } - void deserializeLazyLoadingEntry(HashFuncsMapType::iterator It) const; - - /// Eagerly deserialize all the unloaded entries in the lazy loading map. - void deserializeLazyLoadingEntries() const; - - bool isLazilyLoaded() const { return (bool)Buffer; } - /// A map from a stable_hash to a vector of functions with that hash. - mutable HashFuncsMapType HashToFuncs; + HashFuncsMapType HashToFuncs; /// A vector of strings to hold names. SmallVector IdToName; /// A map from StringRef (name) to an ID. StringMap NameToId; /// True if the function map is finalized with minimal content. bool Finalized = false; - /// The memory buffer that contains the serialized stable function map for - /// lazy loading. - /// Non-empty only if this StableFunctionMap is created from a MemoryBuffer - /// (i.e. by IndexedCodeGenDataReader::read()) and lazily deserialized. - std::shared_ptr Buffer; - /// Whether to read stable function names from the buffer. - bool ReadStableFunctionMapNames = true; friend struct StableFunctionMapRecord; }; diff --git a/llvm/include/llvm/CGData/StableFunctionMapRecord.h b/llvm/include/llvm/CGData/StableFunctionMapRecord.h index 2d8b573a3cb4..a75cb12a70ba 100644 --- a/llvm/include/llvm/CGData/StableFunctionMapRecord.h +++ b/llvm/include/llvm/CGData/StableFunctionMapRecord.h @@ -24,26 +24,6 @@ namespace llvm { -/// The structure of the serialized stable function map is as follows: -/// - Number of unique function/module names -/// - Total size of unique function/module names for opt-in skipping -/// - Unique function/module names -/// - Padding to align to 4 bytes -/// - Number of StableFunctionEntries -/// - Hashes of each StableFunctionEntry -/// - Fixed-size fields for each StableFunctionEntry (the order is consistent -/// with the hashes above): -/// - FunctionNameId -/// - ModuleNameId -/// - InstCount -/// - Relative offset to the beginning of IndexOperandHashes for this entry -/// - Total size of variable-sized IndexOperandHashes for lazy-loading support -/// - Variable-sized IndexOperandHashes for each StableFunctionEntry: -/// - Number of IndexOperandHashes -/// - Contents of each IndexOperandHashes -/// - InstIndex -/// - OpndIndex -/// - OpndHash struct StableFunctionMapRecord { std::unique_ptr FunctionMap; @@ -60,25 +40,13 @@ struct StableFunctionMapRecord { const StableFunctionMap *FunctionMap, std::vector &PatchItems); - /// A static helper function to deserialize the stable function map entry. - /// Ptr should be pointing to the start of the fixed-sized fields of the - /// entry when passed in. - LLVM_ABI static void deserializeEntry(const unsigned char *Ptr, - stable_hash Hash, - StableFunctionMap *FunctionMap); - /// Serialize the stable function map to a raw_ostream. LLVM_ABI void serialize(raw_ostream &OS, std::vector &PatchItems) const; /// Deserialize the stable function map from a raw_ostream. - LLVM_ABI void deserialize(const unsigned char *&Ptr); - - /// Lazily deserialize the stable function map from `Buffer` starting at - /// `Offset`. The individual stable function entry would be read lazily from - /// `Buffer` when the function map is accessed. - LLVM_ABI void lazyDeserialize(std::shared_ptr Buffer, - uint64_t Offset); + LLVM_ABI void deserialize(const unsigned char *&Ptr, + bool ReadStableFunctionMapNames = true); /// Serialize the stable function map to a YAML stream. LLVM_ABI void serializeYAML(yaml::Output &YOS) const; @@ -102,18 +70,6 @@ struct StableFunctionMapRecord { yaml::Output YOS(OS); serializeYAML(YOS); } - - /// Set whether to read stable function names from the buffer. - /// Has no effect if the function map is read from a YAML stream. - void setReadStableFunctionMapNames(bool Read) { - assert( - FunctionMap->empty() && - "Cannot change ReadStableFunctionMapNames after the map is populated"); - FunctionMap->ReadStableFunctionMapNames = Read; - } - -private: - void deserialize(const unsigned char *&Ptr, bool Lazy); }; } // namespace llvm diff --git a/llvm/lib/CGData/CodeGenData.cpp b/llvm/lib/CGData/CodeGenData.cpp index b4f08c3d13b0..cd012342e195 100644 --- a/llvm/lib/CGData/CodeGenData.cpp +++ b/llvm/lib/CGData/CodeGenData.cpp @@ -186,7 +186,7 @@ Expected
Header::readFromBuffer(const unsigned char *Curr) { return make_error(cgdata_error::unsupported_version); H.DataKind = endian::readNext(Curr); - static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version4, + static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version3, "Please update the offset computation below if a new field has " "been added to the header."); H.OutlinedHashTreeOffset = diff --git a/llvm/lib/CGData/CodeGenDataReader.cpp b/llvm/lib/CGData/CodeGenDataReader.cpp index fc59be8df525..0ab35499c898 100644 --- a/llvm/lib/CGData/CodeGenDataReader.cpp +++ b/llvm/lib/CGData/CodeGenDataReader.cpp @@ -26,12 +26,6 @@ static cl::opt IndexedCodeGenDataReadFunctionMapNames( "disabled to save memory and time for final consumption of the " "indexed CodeGenData in production.")); -cl::opt IndexedCodeGenDataLazyLoading( - "indexed-codegen-data-lazy-loading", cl::init(false), cl::Hidden, - cl::desc( - "Lazily load indexed CodeGenData. Enable to save memory and time " - "for final consumption of the indexed CodeGenData in production.")); - namespace llvm { static Expected> @@ -115,20 +109,11 @@ Error IndexedCodeGenDataReader::read() { return error(cgdata_error::eof); HashTreeRecord.deserialize(Ptr); } - - // TODO: lazy loading support for outlined hash tree. - std::shared_ptr SharedDataBuffer = std::move(DataBuffer); if (hasStableFunctionMap()) { const unsigned char *Ptr = Start + Header.StableFunctionMapOffset; if (Ptr >= End) return error(cgdata_error::eof); - FunctionMapRecord.setReadStableFunctionMapNames( - IndexedCodeGenDataReadFunctionMapNames); - if (IndexedCodeGenDataLazyLoading) - FunctionMapRecord.lazyDeserialize(SharedDataBuffer, - Header.StableFunctionMapOffset); - else - FunctionMapRecord.deserialize(Ptr); + FunctionMapRecord.deserialize(Ptr, IndexedCodeGenDataReadFunctionMapNames); } return success(); diff --git a/llvm/lib/CGData/StableFunctionMap.cpp b/llvm/lib/CGData/StableFunctionMap.cpp index 2f54fad0aa08..87f1e76afb60 100644 --- a/llvm/lib/CGData/StableFunctionMap.cpp +++ b/llvm/lib/CGData/StableFunctionMap.cpp @@ -15,10 +15,8 @@ #include "llvm/CGData/StableFunctionMap.h" #include "llvm/ADT/SmallSet.h" -#include "llvm/CGData/StableFunctionMapRecord.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" -#include #define DEBUG_TYPE "stable-function-map" @@ -95,10 +93,9 @@ void StableFunctionMap::insert(const StableFunction &Func) { void StableFunctionMap::merge(const StableFunctionMap &OtherMap) { assert(!Finalized && "Cannot merge after finalization"); - deserializeLazyLoadingEntries(); for (auto &[Hash, Funcs] : OtherMap.HashToFuncs) { - auto &ThisFuncs = HashToFuncs[Hash].Entries; - for (auto &Func : Funcs.Entries) { + auto &ThisFuncs = HashToFuncs[Hash]; + for (auto &Func : Funcs) { auto FuncNameId = getIdOrCreateForName(*OtherMap.getNameForId(Func->FunctionNameId)); auto ModuleNameId = @@ -117,63 +114,25 @@ size_t StableFunctionMap::size(SizeType Type) const { case UniqueHashCount: return HashToFuncs.size(); case TotalFunctionCount: { - deserializeLazyLoadingEntries(); size_t Count = 0; for (auto &Funcs : HashToFuncs) - Count += Funcs.second.Entries.size(); + Count += Funcs.second.size(); return Count; } case MergeableFunctionCount: { - deserializeLazyLoadingEntries(); size_t Count = 0; for (auto &[Hash, Funcs] : HashToFuncs) - if (Funcs.Entries.size() >= 2) - Count += Funcs.Entries.size(); + if (Funcs.size() >= 2) + Count += Funcs.size(); return Count; } } llvm_unreachable("Unhandled size type"); } -const StableFunctionMap::StableFunctionEntries & -StableFunctionMap::at(HashFuncsMapType::key_type FunctionHash) const { - auto It = HashToFuncs.find(FunctionHash); - if (isLazilyLoaded()) - deserializeLazyLoadingEntry(It); - return It->second.Entries; -} - -void StableFunctionMap::deserializeLazyLoadingEntry( - HashFuncsMapType::iterator It) const { - assert(isLazilyLoaded() && "Cannot deserialize non-lazily-loaded map"); - auto &[Hash, Storage] = *It; - std::call_once(Storage.LazyLoadFlag, - [this, HashArg = Hash, &StorageArg = Storage]() { - for (auto Offset : StorageArg.Offsets) - StableFunctionMapRecord::deserializeEntry( - reinterpret_cast(Offset), - HashArg, const_cast(this)); - }); -} - -void StableFunctionMap::deserializeLazyLoadingEntries() const { - if (!isLazilyLoaded()) - return; - for (auto It = HashToFuncs.begin(); It != HashToFuncs.end(); ++It) - deserializeLazyLoadingEntry(It); -} - -const StableFunctionMap::HashFuncsMapType & -StableFunctionMap::getFunctionMap() const { - // Ensure all entries are deserialized before returning the raw map. - if (isLazilyLoaded()) - deserializeLazyLoadingEntries(); - return HashToFuncs; -} - using ParamLocs = SmallVector; -static void -removeIdenticalIndexPair(StableFunctionMap::StableFunctionEntries &SFS) { +static void removeIdenticalIndexPair( + SmallVector> &SFS) { auto &RSF = SFS[0]; unsigned StableFunctionCount = SFS.size(); @@ -200,7 +159,9 @@ removeIdenticalIndexPair(StableFunctionMap::StableFunctionEntries &SFS) { SF->IndexOperandHashMap->erase(Pair); } -static bool isProfitable(const StableFunctionMap::StableFunctionEntries &SFS) { +static bool isProfitable( + const SmallVector> + &SFS) { unsigned StableFunctionCount = SFS.size(); if (StableFunctionCount < GlobalMergingMinMerges) return false; @@ -241,11 +202,8 @@ static bool isProfitable(const StableFunctionMap::StableFunctionEntries &SFS) { } void StableFunctionMap::finalize(bool SkipTrim) { - deserializeLazyLoadingEntries(); - SmallVector ToDelete; for (auto It = HashToFuncs.begin(); It != HashToFuncs.end(); ++It) { - auto &[StableHash, Storage] = *It; - auto &SFS = Storage.Entries; + auto &[StableHash, SFS] = *It; // Group stable functions by ModuleIdentifier. llvm::stable_sort(SFS, [&](const std::unique_ptr &L, @@ -278,7 +236,7 @@ void StableFunctionMap::finalize(bool SkipTrim) { } } if (Invalid) { - ToDelete.push_back(It); + HashToFuncs.erase(It); continue; } @@ -290,10 +248,8 @@ void StableFunctionMap::finalize(bool SkipTrim) { removeIdenticalIndexPair(SFS); if (!isProfitable(SFS)) - ToDelete.push_back(It); + HashToFuncs.erase(It); } - for (auto It : ToDelete) - HashToFuncs.erase(It); Finalized = true; } diff --git a/llvm/lib/CGData/StableFunctionMapRecord.cpp b/llvm/lib/CGData/StableFunctionMapRecord.cpp index e585995ba6a3..423e06802308 100644 --- a/llvm/lib/CGData/StableFunctionMapRecord.cpp +++ b/llvm/lib/CGData/StableFunctionMapRecord.cpp @@ -53,7 +53,7 @@ static SmallVector getStableFunctionEntries(const StableFunctionMap &SFM) { SmallVector FuncEntries; for (const auto &P : SFM.getFunctionMap()) - for (auto &Func : P.second.Entries) + for (auto &Func : P.second) FuncEntries.emplace_back(Func.get()); llvm::stable_sort( @@ -107,25 +107,14 @@ void StableFunctionMapRecord::serialize( // Write StableFunctionEntries whose pointers are sorted. auto FuncEntries = getStableFunctionEntries(*FunctionMap); Writer.write(FuncEntries.size()); - for (const auto *FuncRef : FuncEntries) - Writer.write(FuncRef->Hash); - std::vector IndexOperandHashesOffsets; - IndexOperandHashesOffsets.reserve(FuncEntries.size()); + for (const auto *FuncRef : FuncEntries) { + Writer.write(FuncRef->Hash); Writer.write(FuncRef->FunctionNameId); Writer.write(FuncRef->ModuleNameId); Writer.write(FuncRef->InstCount); - const uint64_t Offset = Writer.OS.tell(); - IndexOperandHashesOffsets.push_back(Offset); - Writer.write(0); - } - const uint64_t IndexOperandHashesByteSizeOffset = Writer.OS.tell(); - Writer.write(0); - for (size_t I = 0; I < FuncEntries.size(); ++I) { - const uint64_t Offset = Writer.OS.tell() - IndexOperandHashesOffsets[I]; - PatchItems.emplace_back(IndexOperandHashesOffsets[I], &Offset, 1); + // Emit IndexOperandHashes sorted from IndexOperandHashMap. - const auto *FuncRef = FuncEntries[I]; IndexOperandHashVecType IndexOperandHashes = getStableIndexOperandHashes(FuncRef); Writer.write(IndexOperandHashes.size()); @@ -135,62 +124,10 @@ void StableFunctionMapRecord::serialize( Writer.write(IndexOperandHash.second); } } - // Write the total size of IndexOperandHashes. - const uint64_t IndexOperandHashesByteSize = - Writer.OS.tell() - IndexOperandHashesByteSizeOffset - sizeof(uint64_t); - PatchItems.emplace_back(IndexOperandHashesByteSizeOffset, - &IndexOperandHashesByteSize, 1); -} - -void StableFunctionMapRecord::deserializeEntry(const unsigned char *Ptr, - stable_hash Hash, - StableFunctionMap *FunctionMap) { - auto FunctionNameId = - endian::readNext(Ptr); - if (FunctionMap->ReadStableFunctionMapNames) - assert(FunctionMap->getNameForId(FunctionNameId) && - "FunctionNameId out of range"); - auto ModuleNameId = - endian::readNext(Ptr); - if (FunctionMap->ReadStableFunctionMapNames) - assert(FunctionMap->getNameForId(ModuleNameId) && - "ModuleNameId out of range"); - auto InstCount = - endian::readNext(Ptr); - - // Read IndexOperandHashes to build IndexOperandHashMap - auto CurrentPosition = reinterpret_cast(Ptr); - auto IndexOperandHashesOffset = - endian::readNext(Ptr); - auto *IndexOperandHashesPtr = reinterpret_cast( - CurrentPosition + IndexOperandHashesOffset); - auto NumIndexOperandHashes = - endian::readNext( - IndexOperandHashesPtr); - auto IndexOperandHashMap = std::make_unique(); - for (unsigned J = 0; J < NumIndexOperandHashes; ++J) { - auto InstIndex = endian::readNext( - IndexOperandHashesPtr); - auto OpndIndex = endian::readNext( - IndexOperandHashesPtr); - auto OpndHash = - endian::readNext( - IndexOperandHashesPtr); - assert(InstIndex < InstCount && "InstIndex out of range"); - - IndexOperandHashMap->try_emplace({InstIndex, OpndIndex}, OpndHash); - } - - // Insert a new StableFunctionEntry into the map. - auto FuncEntry = std::make_unique( - Hash, FunctionNameId, ModuleNameId, InstCount, - std::move(IndexOperandHashMap)); - - FunctionMap->insert(std::move(FuncEntry)); } void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr, - bool Lazy) { + bool ReadStableFunctionMapNames) { // Assert that Ptr is 4-byte aligned assert(((uintptr_t)Ptr % 4) == 0); // Read Names. @@ -202,7 +139,7 @@ void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr, const auto NamesByteSize = endian::readNext(Ptr); const auto NamesOffset = reinterpret_cast(Ptr); - if (FunctionMap->ReadStableFunctionMapNames) { + if (ReadStableFunctionMapNames) { for (unsigned I = 0; I < NumNames; ++I) { StringRef Name(reinterpret_cast(Ptr)); Ptr += Name.size() + 1; @@ -220,51 +157,47 @@ void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr, // Read StableFunctionEntries. auto NumFuncs = endian::readNext(Ptr); - auto FixedSizeFieldsOffset = - reinterpret_cast(Ptr) + NumFuncs * sizeof(stable_hash); - constexpr uint32_t FixedSizeFieldsSizePerEntry = - // FunctionNameId - sizeof(uint32_t) + - // ModuleNameId - sizeof(uint32_t) + - // InstCount - sizeof(uint32_t) + - // Relative offset to IndexOperandHashes - sizeof(uint64_t); for (unsigned I = 0; I < NumFuncs; ++I) { auto Hash = endian::readNext(Ptr); - if (Lazy) { - auto It = FunctionMap->HashToFuncs.try_emplace(Hash).first; - StableFunctionMap::EntryStorage &Storage = It->second; - Storage.Offsets.push_back(FixedSizeFieldsOffset); - } else { - deserializeEntry( - reinterpret_cast(FixedSizeFieldsOffset), Hash, - FunctionMap.get()); + [[maybe_unused]] auto FunctionNameId = + endian::readNext(Ptr); + [[maybe_unused]] auto ModuleNameId = + endian::readNext(Ptr); + // Only validate IDs if we've read the names + if (ReadStableFunctionMapNames) { + assert(FunctionMap->getNameForId(FunctionNameId) && + "FunctionNameId out of range"); + assert(FunctionMap->getNameForId(ModuleNameId) && + "ModuleNameId out of range"); } - FixedSizeFieldsOffset += FixedSizeFieldsSizePerEntry; + + auto InstCount = + endian::readNext(Ptr); + + // Read IndexOperandHashes to build IndexOperandHashMap + auto NumIndexOperandHashes = + endian::readNext(Ptr); + auto IndexOperandHashMap = std::make_unique(); + for (unsigned J = 0; J < NumIndexOperandHashes; ++J) { + auto InstIndex = + endian::readNext(Ptr); + auto OpndIndex = + endian::readNext(Ptr); + auto OpndHash = + endian::readNext(Ptr); + assert(InstIndex < InstCount && "InstIndex out of range"); + + IndexOperandHashMap->try_emplace({InstIndex, OpndIndex}, OpndHash); + } + + // Insert a new StableFunctionEntry into the map. + auto FuncEntry = std::make_unique( + Hash, FunctionNameId, ModuleNameId, InstCount, + std::move(IndexOperandHashMap)); + + FunctionMap->insert(std::move(FuncEntry)); } - - // Update Ptr to the end of the serialized map to meet the expectation of - // CodeGenDataReader. - Ptr = reinterpret_cast(FixedSizeFieldsOffset); - auto IndexOperandHashesByteSize = - endian::readNext(Ptr); - Ptr = reinterpret_cast( - reinterpret_cast(Ptr) + IndexOperandHashesByteSize); -} - -void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr) { - deserialize(Ptr, /*Lazy=*/false); -} - -void StableFunctionMapRecord::lazyDeserialize( - std::shared_ptr Buffer, uint64_t Offset) { - const auto *Ptr = reinterpret_cast( - reinterpret_cast(Buffer->getBufferStart()) + Offset); - deserialize(Ptr, /*Lazy=*/true); - FunctionMap->Buffer = std::move(Buffer); } void StableFunctionMapRecord::serializeYAML(yaml::Output &YOS) const { diff --git a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp index 47640c4aac6d..73f11c1345da 100644 --- a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp +++ b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp @@ -350,8 +350,9 @@ checkConstLocationCompatible(const StableFunctionMap::StableFunctionEntry &SF, return true; } -static ParamLocsVecTy -computeParamInfo(const StableFunctionMap::StableFunctionEntries &SFS) { +static ParamLocsVecTy computeParamInfo( + const SmallVector> + &SFS) { std::map, ParamLocs> HashSeqToLocs; auto &RSF = *SFS[0]; unsigned StableFunctionCount = SFS.size(); @@ -395,18 +396,19 @@ bool GlobalMergeFunc::merge(Module &M, const StableFunctionMap *FunctionMap) { // Collect stable functions related to the current module. DenseMap>> HashToFuncs; + auto &Maps = FunctionMap->getFunctionMap(); for (auto &F : M) { if (!isEligibleFunction(&F)) continue; auto FI = llvm::StructuralHashWithDifferences(F, ignoreOp); - if (FunctionMap->contains(FI.FunctionHash)) + if (Maps.contains(FI.FunctionHash)) HashToFuncs[FI.FunctionHash].emplace_back(&F, std::move(FI)); } for (auto &[Hash, Funcs] : HashToFuncs) { std::optional ParamLocsVec; SmallVector FuncMergeInfos; - auto &SFS = FunctionMap->at(Hash); + auto &SFS = Maps.at(Hash); assert(!SFS.empty()); auto &RFS = SFS[0]; diff --git a/llvm/test/ThinLTO/AArch64/cgdata-merge-write.ll b/llvm/test/ThinLTO/AArch64/cgdata-merge-write.ll index 47042d23cc2c..a4022eb885b4 100644 --- a/llvm/test/ThinLTO/AArch64/cgdata-merge-write.ll +++ b/llvm/test/ThinLTO/AArch64/cgdata-merge-write.ll @@ -36,11 +36,9 @@ ; Merge the cgdata using llvm-cgdata. ; We now validate the content of the merged cgdata. -; Two functions have the same hash with only one different constant at the same location. +; Two functions have the same hash with only one different constnat at a same location. ; RUN: llvm-cgdata --merge -o %tout.cgdata %tout-nowrite.1 %tout-nowrite.2 ; RUN: llvm-cgdata --convert %tout.cgdata -o - | FileCheck %s -; RUN: llvm-cgdata --merge -o %tout-lazy.cgdata %tout-nowrite.1 %tout-nowrite.2 -indexed-codegen-data-lazy-loading -; RUN: llvm-cgdata --convert %tout-lazy.cgdata -indexed-codegen-data-lazy-loading -o - | FileCheck %s ; CHECK: - Hash: [[#%d,HASH:]] ; CHECK-NEXT: FunctionName: f1 diff --git a/llvm/test/tools/llvm-cgdata/empty.test b/llvm/test/tools/llvm-cgdata/empty.test index 2082eca58f07..0d2b0e848a2c 100644 --- a/llvm/test/tools/llvm-cgdata/empty.test +++ b/llvm/test/tools/llvm-cgdata/empty.test @@ -16,7 +16,7 @@ RUN: llvm-cgdata --show %t_emptyheader.cgdata | count 0 # The version number appears when asked, as it's in the header RUN: llvm-cgdata --show --cgdata-version %t_emptyheader.cgdata | FileCheck %s --check-prefix=VERSION -VERSION: Version: 4 +VERSION: Version: 3 # When converting a binary file (w/ the header only) to a text file, it's an empty file as the text format does not have an explicit header. RUN: llvm-cgdata --convert %t_emptyheader.cgdata --format text | count 0 @@ -30,7 +30,7 @@ RUN: llvm-cgdata --convert %t_emptyheader.cgdata --format text | count 0 # uint64_t StableFunctionMapOffset; # } RUN: printf '\xffcgdata\x81' > %t_header.cgdata -RUN: printf '\x04\x00\x00\x00' >> %t_header.cgdata +RUN: printf '\x03\x00\x00\x00' >> %t_header.cgdata RUN: printf '\x00\x00\x00\x00' >> %t_header.cgdata RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata diff --git a/llvm/test/tools/llvm-cgdata/error.test b/llvm/test/tools/llvm-cgdata/error.test index 9484371848a7..92ff484e31ca 100644 --- a/llvm/test/tools/llvm-cgdata/error.test +++ b/llvm/test/tools/llvm-cgdata/error.test @@ -22,9 +22,9 @@ RUN: printf '\xffcgdata\x81' > %t_corrupt.cgdata RUN: not llvm-cgdata --show %t_corrupt.cgdata 2>&1 | FileCheck %s --check-prefix=CORRUPT CORRUPT: {{.}}cgdata: invalid codegen data (file header is corrupt) -# The current version 4 while the header says 5. +# The current version 3 while the header says 4. RUN: printf '\xffcgdata\x81' > %t_version.cgdata -RUN: printf '\x05\x00\x00\x00' >> %t_version.cgdata +RUN: printf '\x04\x00\x00\x00' >> %t_version.cgdata RUN: printf '\x00\x00\x00\x00' >> %t_version.cgdata RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata diff --git a/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test b/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test index 70b83af407e5..b060872113b1 100644 --- a/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test +++ b/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test @@ -23,8 +23,6 @@ RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-both-hashtree-funcma # Merge an object file having cgdata (__llvm_outline and __llvm_merge) RUN: llvm-cgdata -m --skip-trim %t/merge-both-hashtree-funcmap.o -o %t/merge-both-hashtree-funcmap.cgdata RUN: llvm-cgdata -s %t/merge-both-hashtree-funcmap.cgdata | FileCheck %s -RUN: llvm-cgdata -m --skip-trim %t/merge-both-hashtree-funcmap.o -o %t/merge-both-hashtree-funcmap-lazy.cgdata -indexed-codegen-data-lazy-loading -RUN: llvm-cgdata -s %t/merge-both-hashtree-funcmap-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s CHECK: Outlined hash tree: CHECK-NEXT: Total Node Count: 3 @@ -65,4 +63,4 @@ CHECK-NEXT: Mergeable function Count: 0 ;--- merge-both-template.ll @.data1 = private unnamed_addr constant [72 x i8] c"", section "__DATA,__llvm_outline" -@.data2 = private unnamed_addr constant [84 x i8] c"", section "__DATA,__llvm_merge" +@.data2 = private unnamed_addr constant [68 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test index c088ffbb4e83..293608632102 100644 --- a/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test +++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test @@ -23,8 +23,8 @@ RUN: llvm-ar rcs %t/merge-archive.a %t/merge-1.o %t/merge-2.o # Merge the archive into the codegen data file. RUN: llvm-cgdata --merge --skip-trim %t/merge-archive.a -o %t/merge-archive.cgdata RUN: llvm-cgdata --show %t/merge-archive.cgdata | FileCheck %s -RUN: llvm-cgdata --merge --skip-trim %t/merge-archive.a -o %t/merge-archive-lazy.cgdata -indexed-codegen-data-lazy-loading -RUN: llvm-cgdata --show %t/merge-archive-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s + +RUN: llvm-cgdata --show %t/merge-archive.cgdata| FileCheck %s CHECK: Stable function map: CHECK-NEXT: Unique hash Count: 1 CHECK-NEXT: Total function Count: 2 @@ -65,7 +65,7 @@ MAP-NEXT: ... ... ;--- merge-1-template.ll -@.data = private unnamed_addr constant [84 x i8] c"", section "__DATA,__llvm_merge" +@.data = private unnamed_addr constant [68 x i8] c"", section "__DATA,__llvm_merge" ;--- raw-2.cgtext :stable_function_map @@ -80,4 +80,4 @@ MAP-NEXT: ... ... ;--- merge-2-template.ll -@.data = private unnamed_addr constant [84 x i8] c"", section "__DATA,__llvm_merge" +@.data = private unnamed_addr constant [68 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test index 90b5992973b4..d2965456a199 100644 --- a/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test +++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test @@ -17,8 +17,6 @@ RUN: sed "s//$(cat %t/raw-2-bytes.txt)/g" %t/merge-concat-template- RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-concat.ll -o %t/merge-concat.o RUN: llvm-cgdata --merge --skip-trim %t/merge-concat.o -o %t/merge-concat.cgdata RUN: llvm-cgdata --show %t/merge-concat.cgdata | FileCheck %s -RUN: llvm-cgdata --merge --skip-trim %t/merge-concat.o -o %t/merge-concat-lazy.cgdata -indexed-codegen-data-lazy-loading -RUN: llvm-cgdata --show %t/merge-concat-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s CHECK: Stable function map: CHECK-NEXT: Unique hash Count: 1 @@ -76,5 +74,5 @@ MAP-NEXT: ... ; In an linked executable (as opposed to an object file), cgdata in __llvm_merge might be concatenated. ; Although this is not a typical workflow, we simply support this case to parse cgdata that is concatenated. ; In other words, the following two trees are encoded back-to-back in a binary format. -@.data1 = private unnamed_addr constant [84 x i8] c"", section "__DATA,__llvm_merge" -@.data2 = private unnamed_addr constant [84 x i8] c"", section "__DATA,__llvm_merge" +@.data1 = private unnamed_addr constant [68 x i8] c"", section "__DATA,__llvm_merge" +@.data2 = private unnamed_addr constant [68 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test index b986aef26f1d..8277e3272d77 100644 --- a/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test +++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test @@ -19,9 +19,8 @@ RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o # Merge two object files into the codegen data file. RUN: llvm-cgdata --merge --skip-trim %t/merge-1.o %t/merge-2.o -o %t/merge.cgdata + RUN: llvm-cgdata --show %t/merge.cgdata | FileCheck %s -RUN: llvm-cgdata --merge --skip-trim %t/merge-1.o %t/merge-2.o -o %t/merge-lazy.cgdata -indexed-codegen-data-lazy-loading -RUN: llvm-cgdata --show %t/merge-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s CHECK: Stable function map: CHECK-NEXT: Unique hash Count: 1 CHECK-NEXT: Total function Count: 2 @@ -62,7 +61,7 @@ MAP-NEXT: ... ... ;--- merge-1-template.ll -@.data = private unnamed_addr constant [84 x i8] c"", section "__DATA,__llvm_merge" +@.data = private unnamed_addr constant [68 x i8] c"", section "__DATA,__llvm_merge" ;--- raw-2.cgtext :stable_function_map @@ -77,4 +76,4 @@ MAP-NEXT: ... ... ;--- merge-2-template.ll -@.data = private unnamed_addr constant [84 x i8] c"", section "__DATA,__llvm_merge" +@.data = private unnamed_addr constant [68 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test index eac852ff7e71..9469f1cbda33 100644 --- a/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test +++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test @@ -15,8 +15,6 @@ RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-single.ll -o %t/merg # Merge an object file having cgdata (__llvm_merge) RUN: llvm-cgdata -m --skip-trim %t/merge-single.o -o %t/merge-single.cgdata RUN: llvm-cgdata -s %t/merge-single.cgdata | FileCheck %s -RUN: llvm-cgdata -m --skip-trim %t/merge-single.o -o %t/merge-single-lazy.cgdata -indexed-codegen-data-lazy-loading -RUN: llvm-cgdata -s %t/merge-single-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s CHECK: Stable function map: CHECK-NEXT: Unique hash Count: 1 CHECK-NEXT: Total function Count: 1 @@ -35,4 +33,4 @@ CHECK-NEXT: Mergeable function Count: 0 ... ;--- merge-single-template.ll -@.data = private unnamed_addr constant [84 x i8] c"", section "__DATA,__llvm_merge" +@.data = private unnamed_addr constant [68 x i8] c"", section "__DATA,__llvm_merge" diff --git a/llvm/tools/llvm-cgdata/Opts.td b/llvm/tools/llvm-cgdata/Opts.td index 2b515a0140e6..8da933f744e8 100644 --- a/llvm/tools/llvm-cgdata/Opts.td +++ b/llvm/tools/llvm-cgdata/Opts.td @@ -31,4 +31,3 @@ def : JoinedOrSeparate<["-"], "o">, Alias, MetaVarName<"">, HelpTe def format : Option<["--"], "format", KIND_SEPARATE>, HelpText<"Specify the output format (text or binary)">, MetaVarName<"">; def : JoinedOrSeparate<["-"], "f">, Alias, HelpText<"Alias for --format">; -def indexed_codegen_data_lazy_loading : F<"indexed-codegen-data-lazy-loading", "Lazily load indexed CodeGenData for testing purpose.">, Flags<[HelpHidden]>; diff --git a/llvm/tools/llvm-cgdata/llvm-cgdata.cpp b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp index 047557e5a7fa..98fa5c565735 100644 --- a/llvm/tools/llvm-cgdata/llvm-cgdata.cpp +++ b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp @@ -83,8 +83,6 @@ static CGDataAction Action; static std::optional OutputFormat; static std::vector InputFilenames; -extern cl::opt IndexedCodeGenDataLazyLoading; - static void exitWithError(Twine Message, StringRef Whence = "", StringRef Hint = "") { WithColor::error(); @@ -363,9 +361,6 @@ static void parseArgs(int argc, char **argv) { default: llvm_unreachable("unrecognized action"); } - - IndexedCodeGenDataLazyLoading = - Args.hasArg(OPT_indexed_codegen_data_lazy_loading); } int llvm_cgdata_main(int argc, char **argvNonConst, const llvm::ToolContext &) { diff --git a/llvm/unittests/CGData/StableFunctionMapTest.cpp b/llvm/unittests/CGData/StableFunctionMapTest.cpp index 5cf62ae0b394..d551ac8a814f 100644 --- a/llvm/unittests/CGData/StableFunctionMapTest.cpp +++ b/llvm/unittests/CGData/StableFunctionMapTest.cpp @@ -117,7 +117,7 @@ TEST(StableFunctionMap, Finalize3) { Map.finalize(); auto &M = Map.getFunctionMap(); EXPECT_THAT(M, SizeIs(1)); - auto &FuncEntries = M.begin()->second.Entries; + auto &FuncEntries = M.begin()->second; for (auto &FuncEntry : FuncEntries) { EXPECT_THAT(*FuncEntry->IndexOperandHashMap, SizeIs(1)); ASSERT_THAT(*FuncEntry->IndexOperandHashMap,