diff --git a/llvm/include/llvm/ProfileData/IndexedMemProfData.h b/llvm/include/llvm/ProfileData/IndexedMemProfData.h index 304b1c4734af..9af8755281be 100644 --- a/llvm/include/llvm/ProfileData/IndexedMemProfData.h +++ b/llvm/include/llvm/ProfileData/IndexedMemProfData.h @@ -91,6 +91,6 @@ Error writeMemProf( ProfOStream &OS, memprof::IndexedMemProfData &MemProfData, memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema, std::unique_ptr DataAccessProfileData, - memprof::MemProfSummary *MemProfSum); + std::unique_ptr MemProfSum); } // namespace llvm #endif diff --git a/llvm/include/llvm/ProfileData/MemProfCommon.h b/llvm/include/llvm/ProfileData/MemProfCommon.h new file mode 100644 index 000000000000..ff2aeee83436 --- /dev/null +++ b/llvm/include/llvm/ProfileData/MemProfCommon.h @@ -0,0 +1,35 @@ +//===- MemProfCommon.h - MemProf common utilities ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains MemProf common utilities. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_MEMPROFCOMMON_H +#define LLVM_PROFILEDATA_MEMPROFCOMMON_H + +#include "llvm/IR/ModuleSummaryIndex.h" + +namespace llvm { +namespace memprof { + +struct Frame; + +/// Return the allocation type for a given set of memory profile values. +AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity, + uint64_t AllocCount, uint64_t TotalLifetime); + +/// Helper to generate a single hash id for a given callstack, used for emitting +/// matching statistics and useful for uniquing such statistics across modules. +/// Also used to dedup contexts when computing the summary. +uint64_t computeFullStackId(ArrayRef CallStack); + +} // namespace memprof +} // namespace llvm + +#endif // LLVM_PROFILEDATA_MEMPROFCOMMON_H diff --git a/llvm/include/llvm/ProfileData/MemProfSummary.h b/llvm/include/llvm/ProfileData/MemProfSummary.h index 112c2ae9b4b6..f1aa818d9ed7 100644 --- a/llvm/include/llvm/ProfileData/MemProfSummary.h +++ b/llvm/include/llvm/ProfileData/MemProfSummary.h @@ -6,29 +6,18 @@ // //===----------------------------------------------------------------------===// // -// This file contains MemProf summary support and related interfaces. +// This file contains MemProf summary support. // //===----------------------------------------------------------------------===// #ifndef LLVM_PROFILEDATA_MEMPROFSUMMARY_H #define LLVM_PROFILEDATA_MEMPROFSUMMARY_H -#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/ProfileData/InstrProf.h" -#include "llvm/ProfileData/MemProf.h" namespace llvm { namespace memprof { -/// Return the allocation type for a given set of memory profile values. -AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity, - uint64_t AllocCount, uint64_t TotalLifetime); - -/// Helper to generate a single hash id for a given callstack, used for emitting -/// matching statistics and useful for uniquing such statistics across modules. -/// Also used to dedup contexts when computing the summary. -uint64_t computeFullStackId(ArrayRef CallStack); - class MemProfSummary { private: /// The number of summary fields below, which is used to enable some forwards diff --git a/llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h b/llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h index 3c615930d221..c994f823fb94 100644 --- a/llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h +++ b/llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h @@ -26,9 +26,9 @@ private: // instances of the same allocations. DenseSet Contexts; + // Helper called by the public raw and indexed profile addRecord interfaces. void addRecord(uint64_t, const PortableMemInfoBlock &); -protected: uint64_t MaxColdTotalSize = 0; uint64_t MaxWarmTotalSize = 0; uint64_t MaxHotTotalSize = 0; diff --git a/llvm/lib/ProfileData/CMakeLists.txt b/llvm/lib/ProfileData/CMakeLists.txt index b1a0d8d70728..d26af1c3d554 100644 --- a/llvm/lib/ProfileData/CMakeLists.txt +++ b/llvm/lib/ProfileData/CMakeLists.txt @@ -8,6 +8,7 @@ add_llvm_component_library(LLVMProfileData InstrProfWriter.cpp ItaniumManglingCanonicalizer.cpp MemProf.cpp + MemProfCommon.cpp MemProfReader.cpp MemProfRadixTree.cpp MemProfSummary.cpp diff --git a/llvm/lib/ProfileData/IndexedMemProfData.cpp b/llvm/lib/ProfileData/IndexedMemProfData.cpp index cbff51e3cbdc..04ddae357ddd 100644 --- a/llvm/lib/ProfileData/IndexedMemProfData.cpp +++ b/llvm/lib/ProfileData/IndexedMemProfData.cpp @@ -222,7 +222,7 @@ static Error writeMemProfRadixTreeBased( memprof::IndexedVersion Version, bool MemProfFullSchema, std::unique_ptr DataAccessProfileData = nullptr, - memprof::MemProfSummary *MemProfSum = nullptr) { + std::unique_ptr MemProfSum = nullptr) { assert((Version == memprof::Version3 || Version == memprof::Version4) && "Unsupported version for radix tree format"); @@ -303,10 +303,10 @@ static Error writeMemProfV4( ProfOStream &OS, memprof::IndexedMemProfData &MemProfData, bool MemProfFullSchema, std::unique_ptr DataAccessProfileData, - memprof::MemProfSummary *MemProfSum) { + std::unique_ptr MemProfSum) { return writeMemProfRadixTreeBased( OS, MemProfData, memprof::Version4, MemProfFullSchema, - std::move(DataAccessProfileData), MemProfSum); + std::move(DataAccessProfileData), std::move(MemProfSum)); } // Write out the MemProf data in a requested version. @@ -314,7 +314,7 @@ Error writeMemProf( ProfOStream &OS, memprof::IndexedMemProfData &MemProfData, memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema, std::unique_ptr DataAccessProfileData, - memprof::MemProfSummary *MemProfSum) { + std::unique_ptr MemProfSum) { switch (MemProfVersionRequested) { case memprof::Version2: return writeMemProfV2(OS, MemProfData, MemProfFullSchema); @@ -322,7 +322,8 @@ Error writeMemProf( return writeMemProfV3(OS, MemProfData, MemProfFullSchema); case memprof::Version4: return writeMemProfV4(OS, MemProfData, MemProfFullSchema, - std::move(DataAccessProfileData), MemProfSum); + std::move(DataAccessProfileData), + std::move(MemProfSum)); } return make_error( diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 7a4981124762..4d92e9d0ab33 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -618,12 +618,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { if (static_cast(ProfileKind & InstrProfKind::MemProf)) { MemProfSectionStart = OS.tell(); - // Get the finalized MemProf summary that was built when adding records. - auto MemProfSum = MemProfSumBuilder.getSummary(); - if (auto E = writeMemProf( OS, MemProfData, MemProfVersionRequested, MemProfFullSchema, - std::move(DataAccessProfileData), MemProfSum.get())) + std::move(DataAccessProfileData), MemProfSumBuilder.getSummary())) return E; } diff --git a/llvm/lib/ProfileData/MemProfCommon.cpp b/llvm/lib/ProfileData/MemProfCommon.cpp new file mode 100644 index 000000000000..8c0c93123c82 --- /dev/null +++ b/llvm/lib/ProfileData/MemProfCommon.cpp @@ -0,0 +1,81 @@ +//=-- MemProfCommon.cpp - MemProf common utilities ---------------=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains MemProf common utilities. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ProfileData/MemProfCommon.h" +#include "llvm/ProfileData/MemProf.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/HashBuilder.h" + +using namespace llvm; +using namespace llvm::memprof; + +// Upper bound on lifetime access density (accesses per byte per lifetime sec) +// for marking an allocation cold. +cl::opt MemProfLifetimeAccessDensityColdThreshold( + "memprof-lifetime-access-density-cold-threshold", cl::init(0.05), + cl::Hidden, + cl::desc("The threshold the lifetime access density (accesses per byte per " + "lifetime sec) must be under to consider an allocation cold")); + +// Lower bound on lifetime to mark an allocation cold (in addition to accesses +// per byte per sec above). This is to avoid pessimizing short lived objects. +cl::opt MemProfAveLifetimeColdThreshold( + "memprof-ave-lifetime-cold-threshold", cl::init(200), cl::Hidden, + cl::desc("The average lifetime (s) for an allocation to be considered " + "cold")); + +// Lower bound on average lifetime accesses density (total life time access +// density / alloc count) for marking an allocation hot. +cl::opt MemProfMinAveLifetimeAccessDensityHotThreshold( + "memprof-min-ave-lifetime-access-density-hot-threshold", cl::init(1000), + cl::Hidden, + cl::desc("The minimum TotalLifetimeAccessDensity / AllocCount for an " + "allocation to be considered hot")); + +cl::opt + MemProfUseHotHints("memprof-use-hot-hints", cl::init(false), cl::Hidden, + cl::desc("Enable use of hot hints (only supported for " + "unambigously hot allocations)")); + +AllocationType llvm::memprof::getAllocType(uint64_t TotalLifetimeAccessDensity, + uint64_t AllocCount, + uint64_t TotalLifetime) { + // The access densities are multiplied by 100 to hold 2 decimal places of + // precision, so need to divide by 100. + if (((float)TotalLifetimeAccessDensity) / AllocCount / 100 < + MemProfLifetimeAccessDensityColdThreshold + // Lifetime is expected to be in ms, so convert the threshold to ms. + && ((float)TotalLifetime) / AllocCount >= + MemProfAveLifetimeColdThreshold * 1000) + return AllocationType::Cold; + + // The access densities are multiplied by 100 to hold 2 decimal places of + // precision, so need to divide by 100. + if (MemProfUseHotHints && + ((float)TotalLifetimeAccessDensity) / AllocCount / 100 > + MemProfMinAveLifetimeAccessDensityHotThreshold) + return AllocationType::Hot; + + return AllocationType::NotCold; +} + +uint64_t llvm::memprof::computeFullStackId(ArrayRef CallStack) { + llvm::HashBuilder, llvm::endianness::little> + HashBuilder; + for (auto &F : CallStack) + HashBuilder.add(F.Function, F.LineOffset, F.Column); + llvm::BLAKE3Result<8> Hash = HashBuilder.final(); + uint64_t Id; + std::memcpy(&Id, Hash.data(), sizeof(Hash)); + return Id; +} diff --git a/llvm/lib/ProfileData/MemProfSummary.cpp b/llvm/lib/ProfileData/MemProfSummary.cpp index 116eb2f805af..f620b7a74b24 100644 --- a/llvm/lib/ProfileData/MemProfSummary.cpp +++ b/llvm/lib/ProfileData/MemProfSummary.cpp @@ -6,79 +6,15 @@ // //===----------------------------------------------------------------------===// // -// This file contains MemProf summary support and related interfaces. +// This file contains MemProf summary support. // //===----------------------------------------------------------------------===// #include "llvm/ProfileData/MemProfSummary.h" -#include "llvm/Support/BLAKE3.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/HashBuilder.h" using namespace llvm; using namespace llvm::memprof; -// Upper bound on lifetime access density (accesses per byte per lifetime sec) -// for marking an allocation cold. -cl::opt MemProfLifetimeAccessDensityColdThreshold( - "memprof-lifetime-access-density-cold-threshold", cl::init(0.05), - cl::Hidden, - cl::desc("The threshold the lifetime access density (accesses per byte per " - "lifetime sec) must be under to consider an allocation cold")); - -// Lower bound on lifetime to mark an allocation cold (in addition to accesses -// per byte per sec above). This is to avoid pessimizing short lived objects. -cl::opt MemProfAveLifetimeColdThreshold( - "memprof-ave-lifetime-cold-threshold", cl::init(200), cl::Hidden, - cl::desc("The average lifetime (s) for an allocation to be considered " - "cold")); - -// Lower bound on average lifetime accesses density (total life time access -// density / alloc count) for marking an allocation hot. -cl::opt MemProfMinAveLifetimeAccessDensityHotThreshold( - "memprof-min-ave-lifetime-access-density-hot-threshold", cl::init(1000), - cl::Hidden, - cl::desc("The minimum TotalLifetimeAccessDensity / AllocCount for an " - "allocation to be considered hot")); - -cl::opt - MemProfUseHotHints("memprof-use-hot-hints", cl::init(false), cl::Hidden, - cl::desc("Enable use of hot hints (only supported for " - "unambigously hot allocations)")); - -AllocationType llvm::memprof::getAllocType(uint64_t TotalLifetimeAccessDensity, - uint64_t AllocCount, - uint64_t TotalLifetime) { - // The access densities are multiplied by 100 to hold 2 decimal places of - // precision, so need to divide by 100. - if (((float)TotalLifetimeAccessDensity) / AllocCount / 100 < - MemProfLifetimeAccessDensityColdThreshold - // Lifetime is expected to be in ms, so convert the threshold to ms. - && ((float)TotalLifetime) / AllocCount >= - MemProfAveLifetimeColdThreshold * 1000) - return AllocationType::Cold; - - // The access densities are multiplied by 100 to hold 2 decimal places of - // precision, so need to divide by 100. - if (MemProfUseHotHints && - ((float)TotalLifetimeAccessDensity) / AllocCount / 100 > - MemProfMinAveLifetimeAccessDensityHotThreshold) - return AllocationType::Hot; - - return AllocationType::NotCold; -} - -uint64_t llvm::memprof::computeFullStackId(ArrayRef CallStack) { - llvm::HashBuilder, llvm::endianness::little> - HashBuilder; - for (auto &F : CallStack) - HashBuilder.add(F.Function, F.LineOffset, F.Column); - llvm::BLAKE3Result<8> Hash = HashBuilder.final(); - uint64_t Id; - std::memcpy(&Id, Hash.data(), sizeof(Hash)); - return Id; -} - void MemProfSummary::printSummaryYaml(raw_ostream &OS) const { // For now emit as YAML comments, since they aren't read on input. OS << "---\n"; diff --git a/llvm/lib/ProfileData/MemProfSummaryBuilder.cpp b/llvm/lib/ProfileData/MemProfSummaryBuilder.cpp index 591bcb6b8a3e..2faf49ee3ec2 100644 --- a/llvm/lib/ProfileData/MemProfSummaryBuilder.cpp +++ b/llvm/lib/ProfileData/MemProfSummaryBuilder.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ProfileData/MemProfSummaryBuilder.h" +#include "llvm/ProfileData/MemProfCommon.h" using namespace llvm; using namespace llvm::memprof; diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp index deb9f8999eab..707226bed851 100644 --- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp @@ -36,7 +36,7 @@ #include "llvm/IR/Value.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/InstrProfReader.h" -#include "llvm/ProfileData/MemProfSummary.h" +#include "llvm/ProfileData/MemProfCommon.h" #include "llvm/Support/BLAKE3.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp index ee06c2391e20..1dbafea19573 100644 --- a/llvm/unittests/ProfileData/MemProfTest.cpp +++ b/llvm/unittests/ProfileData/MemProfTest.cpp @@ -15,22 +15,20 @@ #include "llvm/IR/Value.h" #include "llvm/Object/ObjectFile.h" #include "llvm/ProfileData/IndexedMemProfData.h" +#include "llvm/ProfileData/MemProfCommon.h" #include "llvm/ProfileData/MemProfData.inc" #include "llvm/ProfileData/MemProfRadixTree.h" #include "llvm/ProfileData/MemProfReader.h" -#include "llvm/ProfileData/MemProfSummary.h" #include "llvm/Support/raw_ostream.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include -using namespace llvm; - -extern cl::opt MemProfLifetimeAccessDensityColdThreshold; -extern cl::opt MemProfAveLifetimeColdThreshold; -extern cl::opt MemProfMinAveLifetimeAccessDensityHotThreshold; -extern cl::opt MemProfUseHotHints; +extern llvm::cl::opt MemProfLifetimeAccessDensityColdThreshold; +extern llvm::cl::opt MemProfAveLifetimeColdThreshold; +extern llvm::cl::opt MemProfMinAveLifetimeAccessDensityHotThreshold; +extern llvm::cl::opt MemProfUseHotHints; namespace llvm { namespace memprof {