00cd6c0420
Recently we observed high memory pressure caused by clang during some parallel builds. We discovered that we have several projects that have a large number of #define directives in their TUs (on the order of millions), which caused huge memory consumption in clang due to a lot of allocations for MacroInfo. We would like to reduce the memory overhead of clang for a single #define to reduce the memory overhead for these files, to allow us to reduce the memory pressure on the system during highly parallel builds. This change achieves that by removing the SmallVector in MacroInfo and instead storing the tokens in an array allocated using the bump pointer allocator, after all tokens are lexed. The added unit test with 1000000 #define directives illustrates the problem. Prior to this change, on arm64 macOS, clang's PP bump pointer allocator allocated 272007616 bytes, and used roughly 272 bytes per #define. After this change, clang's PP bump pointer allocator allocates 120002016 bytes, and uses only roughly 120 bytes per #define. For an example test file that we have internally with 7.8 million #define directives, this change produces the following improvement on arm64 macOS: Persistent allocation footprint for this test case file as it's being compiled to LLVM IR went down 22% from 5.28 GB to 4.07 GB and the total allocations went down 14% from 8.26 GB to 7.05 GB. Furthermore, this change reduced the total number of allocations made by the system for this clang invocation from 1454853 to 133663, an order of magnitude improvement. The recommit fixes the LLDB build failure. Differential Revision: https://reviews.llvm.org/D117348
98 lines
3.2 KiB
C++
98 lines
3.2 KiB
C++
//===- unittests/Lex/PPMemoryAllocationsTest.cpp - ----------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===--------------------------------------------------------------===//
|
|
|
|
#include "clang/Basic/Diagnostic.h"
|
|
#include "clang/Basic/DiagnosticOptions.h"
|
|
#include "clang/Basic/FileManager.h"
|
|
#include "clang/Basic/LangOptions.h"
|
|
#include "clang/Basic/SourceManager.h"
|
|
#include "clang/Basic/TargetInfo.h"
|
|
#include "clang/Basic/TargetOptions.h"
|
|
#include "clang/Lex/HeaderSearch.h"
|
|
#include "clang/Lex/HeaderSearchOptions.h"
|
|
#include "clang/Lex/ModuleLoader.h"
|
|
#include "clang/Lex/Preprocessor.h"
|
|
#include "clang/Lex/PreprocessorOptions.h"
|
|
#include "gtest/gtest.h"
|
|
|
|
using namespace clang;
|
|
|
|
namespace {
|
|
|
|
class PPMemoryAllocationsTest : public ::testing::Test {
|
|
protected:
|
|
PPMemoryAllocationsTest()
|
|
: FileMgr(FileMgrOpts), DiagID(new DiagnosticIDs()),
|
|
Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
|
|
SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions) {
|
|
TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
|
|
Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
|
|
}
|
|
|
|
FileSystemOptions FileMgrOpts;
|
|
FileManager FileMgr;
|
|
IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
|
|
DiagnosticsEngine Diags;
|
|
SourceManager SourceMgr;
|
|
LangOptions LangOpts;
|
|
std::shared_ptr<TargetOptions> TargetOpts;
|
|
IntrusiveRefCntPtr<TargetInfo> Target;
|
|
};
|
|
|
|
TEST_F(PPMemoryAllocationsTest, PPMacroDefinesAllocations) {
|
|
std::string Source;
|
|
size_t NumMacros = 1000000;
|
|
{
|
|
llvm::raw_string_ostream SourceOS(Source);
|
|
|
|
// Create a combination of 1 or 3 token macros.
|
|
for (size_t I = 0; I < NumMacros; ++I) {
|
|
SourceOS << "#define MACRO_ID_" << I << " ";
|
|
if ((I % 2) == 0)
|
|
SourceOS << "(" << I << ")";
|
|
else
|
|
SourceOS << I;
|
|
SourceOS << "\n";
|
|
}
|
|
}
|
|
|
|
std::unique_ptr<llvm::MemoryBuffer> Buf =
|
|
llvm::MemoryBuffer::getMemBuffer(Source);
|
|
SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
|
|
|
|
TrivialModuleLoader ModLoader;
|
|
HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
|
|
Diags, LangOpts, Target.get());
|
|
Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
|
|
SourceMgr, HeaderInfo, ModLoader,
|
|
/*IILookup =*/nullptr,
|
|
/*OwnsHeaderSearch =*/false);
|
|
PP.Initialize(*Target);
|
|
PP.EnterMainSourceFile();
|
|
|
|
while (1) {
|
|
Token tok;
|
|
PP.Lex(tok);
|
|
if (tok.is(tok::eof))
|
|
break;
|
|
}
|
|
|
|
size_t NumAllocated = PP.getPreprocessorAllocator().getBytesAllocated();
|
|
float BytesPerDefine = float(NumAllocated) / float(NumMacros);
|
|
llvm::errs() << "Num preprocessor allocations for " << NumMacros
|
|
<< " #define: " << NumAllocated << "\n";
|
|
llvm::errs() << "Bytes per #define: " << BytesPerDefine << "\n";
|
|
// On arm64-apple-macos, we get around 120 bytes per define.
|
|
// Assume a reasonable upper bound based on that number that we don't want
|
|
// to exceed when storing information about a macro #define with 1 or 3
|
|
// tokens.
|
|
EXPECT_LT(BytesPerDefine, 130.0f);
|
|
}
|
|
|
|
} // anonymous namespace
|