a6af641b89
This patch replaces the previous `libcxx-compare-benchmarks` wrapper by a new `compare-benchmarks` script which works with LNT-compatible data. This allows comparing benchmark results across libc++ microbenchmarks, SPEC, and anything else that would produce LNT-compatible data. It also adds a simple script to consolidate LNT benchmark output into a single file, simplifying the process of doing A/B runs locally. The simplest way to do this doesn't require creating two build directories after this patch anymore. It also adds the ability to produce either a standalone HTML chart or a plain text output for diffing results locally when prototyping changes. Example text output of the new tool: ``` Benchmark Baseline Candidate Difference % Difference ----------------------------------- ---------- ----------- ------------ -------------- BM_join_view_deques/0 8.11 8.16 0.05 0.63 BM_join_view_deques/1 13.56 13.79 0.23 1.69 BM_join_view_deques/1024 6606.51 7011.34 404.83 6.13 BM_join_view_deques/2 17.99 19.92 1.93 10.72 BM_join_view_deques/4000 27655.58 29864.72 2209.14 7.99 BM_join_view_deques/4096 26218.07 30520.13 4302.05 16.41 BM_join_view_deques/512 3231.66 2832.47 -399.19 -12.35 BM_join_view_deques/5500 47144.82 42207.41 -4937.42 -10.47 BM_join_view_deques/64 247.23 262.66 15.43 6.24 BM_join_view_deques/64000 756221.63 511247.48 -244974.15 -32.39 BM_join_view_deques/65536 537110.91 560241.61 23130.70 4.31 BM_join_view_deques/70000 815739.07 616181.34 -199557.73 -24.46 BM_join_view_out_vectors/0 0.93 0.93 0.00 0.07 BM_join_view_out_vectors/1 3.11 3.14 0.03 0.82 BM_join_view_out_vectors/1024 3090.92 3563.29 472.37 15.28 BM_join_view_out_vectors/2 5.52 5.56 0.04 0.64 BM_join_view_out_vectors/4000 9887.21 9774.40 -112.82 -1.14 BM_join_view_out_vectors/4096 10158.78 10190.44 31.66 0.31 BM_join_view_out_vectors/512 1218.68 1209.59 -9.09 -0.75 BM_join_view_out_vectors/5500 13559.23 13676.06 116.84 0.86 BM_join_view_out_vectors/64 158.95 157.91 -1.04 -0.65 BM_join_view_out_vectors/64000 178514.73 226520.97 48006.24 26.89 BM_join_view_out_vectors/65536 184639.37 207180.35 22540.98 12.21 BM_join_view_out_vectors/70000 235006.69 213886.93 -21119.77 -8.99 ```
37 lines
1.2 KiB
Python
Executable File
37 lines
1.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import pathlib
|
|
import sys
|
|
|
|
def main(argv):
|
|
parser = argparse.ArgumentParser(
|
|
prog='consolidate-benchmarks',
|
|
description='Consolidate benchmark result files (in LNT format) into a single LNT-format file.')
|
|
parser.add_argument('files_or_directories', type=str, nargs='+',
|
|
help='Path to files or directories containing LNT data to consolidate. Directories are searched '
|
|
'recursively for files with a .lnt extension.')
|
|
parser.add_argument('--output', '-o', type=argparse.FileType('w'), default=sys.stdout,
|
|
help='Where to output the result. Default to stdout.')
|
|
args = parser.parse_args(argv)
|
|
|
|
files = []
|
|
for arg in args.files_or_directories:
|
|
path = pathlib.Path(arg)
|
|
if path.is_dir():
|
|
for p in path.rglob('*.lnt'):
|
|
files.append(p)
|
|
else:
|
|
files.append(path)
|
|
|
|
for file in files:
|
|
for line in file.open().readlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
args.output.write(line)
|
|
args.output.write('\n')
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1:])
|