From 7ce6a94c61d030e60db998bce2142f064aac20f4 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Tue, 20 Jan 2026 09:57:37 -0500 Subject: [PATCH] [libc++] Add a script to produce benchmarks for LNT (#175594) This patch adds a script to run a subset of libc++'s benchmarks for uploading to LNT. As part of this patch the test-at-commit script is modified to no longer build the library itself. Indeed, this provides the necessary flexibility to run the test suite multiple times on the same built library, and also addresses previous concerns where test-at-commit couldn't customize how the library is being built. --- libcxx/utils/benchmark-historical | 49 +++++----- libcxx/utils/build-at-commit | 2 +- libcxx/utils/ci/benchmark-for-lnt.py | 128 +++++++++++++++++++++++++++ libcxx/utils/test-at-commit | 94 +++++++++++++++----- 4 files changed, 228 insertions(+), 45 deletions(-) create mode 100755 libcxx/utils/ci/benchmark-for-lnt.py diff --git a/libcxx/utils/benchmark-historical b/libcxx/utils/benchmark-historical index c1f9d11a6e80..8a315be522d5 100755 --- a/libcxx/utils/benchmark-historical +++ b/libcxx/utils/benchmark-historical @@ -76,30 +76,37 @@ def main(argv): logging.info(f'Skipping {commit} which already has data in {output_file}') continue else: - logging.info(f'Benchmarking {commit}') + logging.info(f'Benchmarking {commit} against test-suite in {args.git_repo}') - with tempfile.TemporaryDirectory() as build_dir: - test_cmd = [PARENT_DIR / 'test-at-commit', '--git-repo', args.git_repo, - '--build', build_dir, - '--commit', commit] - test_cmd += ['--'] + lit_options + with tempfile.TemporaryDirectory() as libcxx_install_dir: + with tempfile.TemporaryDirectory() as build_dir: + build_cmd = [PARENT_DIR / 'build-at-commit', '--git-repo', args.git_repo, + '--commit', commit, + '--install-dir', libcxx_install_dir, + '--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo'] - if args.dry_run: - pretty = ' '.join(str(a) for a in test_cmd) - logging.info(f'Running {pretty}') - continue + test_cmd = [PARENT_DIR / 'test-at-commit', '--git-repo', args.git_repo, + '--libcxx-installation', libcxx_install_dir, + '--build-dir', build_dir] + test_cmd += ['--'] + lit_options - subprocess.call(test_cmd) - output_file.parent.mkdir(parents=True, exist_ok=True) - mode = 'a' if args.existing == 'append' else 'w' - if output_file.exists() and args.existing == 'append': - logging.info(f'Appending to existing data for {commit}') - elif output_file.exists() and args.existing == 'overwrite': - logging.info(f'Overwriting existing data for {commit}') - else: - logging.info(f'Writing data for {commit}') - with open(output_file, mode) as out: - subprocess.check_call([(PARENT_DIR / 'consolidate-benchmarks'), build_dir], stdout=out) + if args.dry_run: + logging.info(f'Running {" ".join(str(a) for a in build_cmd)}') + logging.info(f'Running {" ".join(str(a) for a in test_cmd)}') + continue + + subprocess.check_call(build_cmd) + subprocess.call(test_cmd) + output_file.parent.mkdir(parents=True, exist_ok=True) + mode = 'a' if args.existing == 'append' else 'w' + if output_file.exists() and args.existing == 'append': + logging.info(f'Appending to existing data for {commit}') + elif output_file.exists() and args.existing == 'overwrite': + logging.info(f'Overwriting existing data for {commit}') + else: + logging.info(f'Writing data for {commit}') + with open(output_file, mode) as out: + subprocess.check_call([(PARENT_DIR / 'consolidate-benchmarks'), build_dir], stdout=out) if __name__ == '__main__': main(sys.argv[1:]) diff --git a/libcxx/utils/build-at-commit b/libcxx/utils/build-at-commit index 8af7d1161f70..2b741e3501a5 100755 --- a/libcxx/utils/build-at-commit +++ b/libcxx/utils/build-at-commit @@ -98,7 +98,7 @@ def main(argv): # Gather CMake options cmake_options = [] - if args.cmake_options is not None: + if args.cmake_options: if args.cmake_options[0] != '--': raise ArgumentError('For clarity, CMake options must be separated from other options by --') cmake_options = args.cmake_options[1:] diff --git a/libcxx/utils/ci/benchmark-for-lnt.py b/libcxx/utils/ci/benchmark-for-lnt.py new file mode 100755 index 000000000000..3105010ebe88 --- /dev/null +++ b/libcxx/utils/ci/benchmark-for-lnt.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +import argparse +import os +import pathlib +import subprocess +import sys +import tempfile + +def step(message: str) -> None: + print(message, file=sys.stderr) + +def directory_path(string): + if os.path.isdir(string): + return pathlib.Path(string) + else: + raise NotADirectoryError(string) + +def main(argv): + parser = argparse.ArgumentParser( + prog='benchmark-for-lnt', + description='Benchmark libc++ at the given commit for submitting to LNT.') + parser.add_argument('-o', '--output', type=argparse.FileType('w'), default='-', + help='Path to the file where the resulting LNT report containing benchmark results is written. ' + 'By default, stdout.') + parser.add_argument('--benchmark-commit', type=str, required=True, + help='The SHA representing the version of the library to benchmark.') + parser.add_argument('--test-suite-commit', type=str, required=True, + help='The SHA representing the version of the test suite to use for benchmarking.') + parser.add_argument('--machine', type=str, required=True, + help='The name of the machine for reporting LNT results.') + parser.add_argument('--spec-dir', type=pathlib.Path, required=False, + help='Optional path to a SPEC installation to use for benchmarking.') + parser.add_argument('--git-repo', type=directory_path, default=os.getcwd(), + help='Optional path to the Git repository to use. By default, the current working directory is used.') + parser.add_argument('--dry-run', action='store_true', + help='Only print what would be executed.') + parser.add_argument('-v', '--verbose', action='store_true', + help='Print the output of all subcommands.') + args = parser.parse_args(argv) + + def run(command, *posargs, **kwargs): + command = [str(c) for c in command] + if args.dry_run: + print(f'$ {" ".join(command)}') + else: + # If we're running with verbose, print everything but redirect output to stderr since + # we already output the json to stdout in some cases. Otherwise, hush everything. + if args.verbose: + if 'stdout' not in kwargs: + kwargs.update({'stdout': sys.stderr}) + else: + if 'stdout' not in kwargs: + kwargs.update({'stdout': subprocess.DEVNULL}) + if 'stderr' not in kwargs: + kwargs.update({'stderr': subprocess.DEVNULL}) + subprocess.check_call(command, *posargs, **kwargs) + + with tempfile.TemporaryDirectory() as build_dir: + build_dir = pathlib.Path(build_dir) + + step(f'Building libc++ at commit {args.benchmark_commit}') + run([args.git_repo / 'libcxx/utils/build-at-commit', + '--git-repo', args.git_repo, + '--install-dir', build_dir / 'install', + '--commit', args.benchmark_commit, + '--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo']) + + if args.spec_dir is not None: + step(f'Running SPEC benchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}') + run([args.git_repo / 'libcxx/utils/test-at-commit', + '--git-repo', args.git_repo, + '--build-dir', build_dir / 'spec', + '--test-suite-commit', args.test_suite_commit, + '--libcxx-installation', build_dir / 'install', + '--', + '-j1', '--time-tests', + '--param', 'optimization=speed', + '--param', 'std=c++17', + '--param', f'spec_dir={args.spec_dir}', + build_dir / 'spec/libcxx/test', + '--filter', 'benchmarks/spec.gen.py']) + + # TODO: For now, we run only a subset of the benchmarks because running the whole test suite is too slow. + # Run the whole test suite once https://github.com/llvm/llvm-project/issues/173032 is resolved. + step(f'Running microbenchmarks from {args.test_suite_commit} against libc++ {args.benchmark_commit}') + run([args.git_repo / 'libcxx/utils/test-at-commit', + '--git-repo', args.git_repo, + '--build-dir', build_dir / 'micro', + '--test-suite-commit', args.test_suite_commit, + '--libcxx-installation', build_dir / 'install', + '--', + '-j1', '--time-tests', + '--param', 'optimization=speed', + '--param', 'std=c++26', + build_dir / 'micro/libcxx/test', + '--filter', 'benchmarks/(algorithms|containers|iterators|locale|memory|streams|numeric|utility)']) + + step('Installing LNT') + run(['python', '-m', 'venv', build_dir / '.venv']) + run([build_dir / '.venv/bin/pip', 'install', 'llvm-lnt']) + + step('Consolidating benchmark results and creating JSON report') + if args.spec_dir is not None: + with open(build_dir / 'benchmarks.lnt', 'w') as f: + run([args.git_repo / 'libcxx/utils/consolidate-benchmarks', build_dir / 'spec'], stdout=f) + with open(build_dir / 'benchmarks.lnt', 'a') as f: + run([args.git_repo / 'libcxx/utils/consolidate-benchmarks', build_dir / 'micro'], stdout=f) + order = len(subprocess.check_output(['git', '-C', args.git_repo, 'rev-list', args.benchmark_commit]).splitlines()) + commit_info = subprocess.check_output(['git', '-C', args.git_repo, 'show', args.benchmark_commit, '--no-patch']).decode() + run([build_dir / '.venv/bin/lnt', 'importreport', '--order', str(order), '--machine', args.machine, + '--run-info', f'commit_info={commit_info}', + build_dir / 'benchmarks.lnt', build_dir / 'benchmarks.json']) + + if not args.dry_run: + with open(build_dir / 'benchmarks.json', 'r') as f: + args.output.write(f.read()) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/libcxx/utils/test-at-commit b/libcxx/utils/test-at-commit index f20bf5fd4b10..d1b33c243b4d 100755 --- a/libcxx/utils/test-at-commit +++ b/libcxx/utils/test-at-commit @@ -35,6 +35,41 @@ libcxx.test.config.configure( ) """ +# Unofficial list of directories required to build libc++. This is a best guess that should work +# when checking out the monorepo at most commits, but it's technically not guaranteed to work +# (especially for much older commits). +LIBCXX_REQUIRED_DIRECTORIES = [ + 'libcxx', + 'libcxxabi', + 'llvm/cmake', + 'llvm/utils/llvm-lit', + 'llvm/utils/lit', + 'runtimes', + 'cmake', + 'third-party/benchmark', + 'libc' +] + +def checkout_subdirectories(git_repo, commit, paths, destination): + """ + Produce a copy of the specified Git-tracked files/directories at the given commit. + The resulting files and directories at placed at the given location. + """ + with tempfile.TemporaryDirectory() as tmp: + tmpfile = os.path.join(tmp, 'archive.tar.gz') + git_archive = ['git', '-C', git_repo, 'archive', '--format', 'tar.gz', '--output', tmpfile, commit, '--'] + list(paths) + subprocess.check_call(git_archive) + os.makedirs(destination, exist_ok=True) + subprocess.check_call(['tar', '-x', '-z', '-f', tmpfile, '-C', destination]) + +def exists_in_commit(git_repo, commit, path): + """ + Return whether the given path (file or directory) existed at the given commit. + """ + cmd = ['git', '-C', git_repo, 'show', f'{commit}:{path}'] + result = subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return result == 0 + def directory_path(string): if os.path.isdir(string): return pathlib.Path(string) @@ -44,16 +79,18 @@ def directory_path(string): def main(argv): parser = argparse.ArgumentParser( prog='test-at-commit', - description='Build libc++ at the specified commit and test it against the version of the test suite ' - 'currently checked out in the specified Git repository. ' - 'This makes it easier to perform historical analyses of libc++ behavior, gather historical ' - 'performance data, bisect issues, and so on. ' - 'A current limitation of this script is that it assumes the arguments passed to CMake when ' - 'building the library.') - parser.add_argument('--build', '-B', type=pathlib.Path, required=True, - help='Path to create the build directory for running the test suite at.') - parser.add_argument('--commit', type=str, required=True, - help='Commit to build libc++ at.') + description='Test the provided libc++ installation against the test suite at the specified commit (or ' + 'the currently checked-out sources by default). This makes it easier to perform historical ' + 'analyses of libc++ behavior, gather historical performance data, bisect issues, and so on.') + parser.add_argument('--build-dir', '-B', type=pathlib.Path, required=True, + help='Path to create the build directory for running the test suite at. The results of the tests ' + 'are located in that directory after the run.') + parser.add_argument('--libcxx-installation', type=pathlib.Path, required=True, + help='Path to the directory where a copy of libc++ to run tests on is installed.') + parser.add_argument('--test-suite-commit', type=str, required=False, + help='Commit to use for the test suite. If left unspecified, the currently checked-out version of the ' + 'test suite is used. Otherwise, the requested version is checked out in a separate directory and ' + 'that version of the test suite is used.') parser.add_argument('lit_options', nargs=argparse.REMAINDER, help='Optional arguments passed to lit when running the tests. Should be provided last and ' 'separated from other arguments with a `--`.') @@ -61,6 +98,9 @@ def main(argv): help='Optional path to the Git repository to use. By default, the current working directory is used.') args = parser.parse_args(argv) + args.build_dir = args.build_dir.resolve() + args.libcxx_installation = args.libcxx_installation.resolve() + # Gather lit options lit_options = [] if args.lit_options is not None: @@ -68,29 +108,37 @@ def main(argv): raise ArgumentError('For clarity, Lit options must be separated from other options by --') lit_options = args.lit_options[1:] - with tempfile.TemporaryDirectory() as install_dir: - # Build the library at the baseline - build_cmd = [PARENT_DIR / 'build-at-commit', '--git-repo', args.git_repo, - '--install-dir', install_dir, - '--commit', args.commit] - build_cmd += ['--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo'] - subprocess.check_call(build_cmd) + # This is the list of directories that must be cleaned up before we return + tempdirs = [] + try: + # If needed, check out the test suite at the commit we're going to use for the suite + if args.test_suite_commit is None: + test_suite_sources = args.git_repo + else: + tempdirs.append(tempfile.TemporaryDirectory()) + test_suite_sources = pathlib.Path(tempdirs[-1].name) + checkout_dirs = [d for d in LIBCXX_REQUIRED_DIRECTORIES if exists_in_commit(args.git_repo, args.test_suite_commit, d)] + checkout_subdirectories(args.git_repo, args.test_suite_commit, checkout_dirs, test_suite_sources) # Configure the test suite in the specified build directory - args.build.mkdir(parents=True, exist_ok=True) - lit_cfg = (args.build / 'temp_lit_cfg.cfg.in').absolute() + args.build_dir.mkdir(parents=True, exist_ok=True) + lit_cfg = (args.build_dir / 'temp_lit_cfg.cfg.in').absolute() with open(lit_cfg, 'w') as f: - f.write(LIT_CONFIG_FILE.format(INSTALL_ROOT=install_dir)) + f.write(LIT_CONFIG_FILE.format(INSTALL_ROOT=args.libcxx_installation)) - test_suite_cmd = ['cmake', '-B', args.build, '-S', args.git_repo / 'runtimes', '-G', 'Ninja'] + test_suite_cmd = ['cmake', '-B', args.build_dir, '-S', test_suite_sources / 'runtimes', '-G', 'Ninja'] test_suite_cmd += ['-D', 'LLVM_ENABLE_RUNTIMES=libcxx;libcxxabi'] test_suite_cmd += ['-D', 'LIBCXXABI_USE_LLVM_UNWINDER=OFF'] test_suite_cmd += ['-D', f'LIBCXX_TEST_CONFIG={lit_cfg}'] subprocess.check_call(test_suite_cmd) - # Run the specified tests against the produced baseline installation - lit_cmd = [PARENT_DIR / 'libcxx-lit', args.build] + lit_options + # Run the specified tests against the built library + lit_cmd = [PARENT_DIR / 'libcxx-lit', args.build_dir] + lit_options subprocess.check_call(lit_cmd) + finally: + for d in tempdirs: + d.cleanup() + if __name__ == '__main__': main(sys.argv[1:])