aec12c1264
Summary: Whith the number of projects growing, it is important to be able to filter them in a more convenient way than by names. It is especially important for benchmarks, when it is not viable to analyze big projects 20 or 50 times in a row. Because of this reason, this commit adds a notion of sizes and a filtering interface that puts a limit on a maximum size of the project to analyze or benchmark. Sizes assigned to the projects in this commit, do not directly correspond to the number of lines or files in the project. The key factor that is important for the developers of the analyzer is the time it takes to analyze the project. And for this very reason, "size" basically helps to cluster projects based on their analysis time. Differential Revision: https://reviews.llvm.org/D83942
361 lines
15 KiB
Python
Executable File
361 lines
15 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import argparse
|
|
import sys
|
|
import os
|
|
|
|
from subprocess import call
|
|
|
|
SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
|
|
DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR,
|
|
os.path.pardir,
|
|
os.path.pardir,
|
|
os.path.pardir))
|
|
|
|
|
|
def add(parser, args):
|
|
import SATestAdd
|
|
from ProjectMap import ProjectInfo
|
|
|
|
if args.source == "git" and (args.origin == "" or args.commit == ""):
|
|
parser.error(
|
|
"Please provide both --origin and --commit if source is 'git'")
|
|
|
|
if args.source != "git" and (args.origin != "" or args.commit != ""):
|
|
parser.error("Options --origin and --commit don't make sense when "
|
|
"source is not 'git'")
|
|
|
|
project = ProjectInfo(args.name[0], args.mode, args.source, args.origin,
|
|
args.commit)
|
|
|
|
SATestAdd.add_new_project(project)
|
|
|
|
|
|
def build(parser, args):
|
|
import SATestBuild
|
|
|
|
SATestBuild.VERBOSE = args.verbose
|
|
|
|
projects = get_projects(parser, args)
|
|
tester = SATestBuild.RegressionTester(args.jobs,
|
|
projects,
|
|
args.override_compiler,
|
|
args.extra_analyzer_config,
|
|
args.regenerate,
|
|
args.strictness)
|
|
tests_passed = tester.test_all()
|
|
|
|
if not tests_passed:
|
|
sys.stderr.write("ERROR: Tests failed.\n")
|
|
sys.exit(42)
|
|
|
|
|
|
def compare(parser, args):
|
|
import CmpRuns
|
|
|
|
choices = [CmpRuns.HistogramType.RELATIVE.value,
|
|
CmpRuns.HistogramType.LOG_RELATIVE.value,
|
|
CmpRuns.HistogramType.ABSOLUTE.value]
|
|
|
|
if args.histogram is not None and args.histogram not in choices:
|
|
parser.error("Incorrect histogram type, available choices are {}"
|
|
.format(choices))
|
|
|
|
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
|
|
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
|
|
|
|
CmpRuns.dump_scan_build_results_diff(dir_old, dir_new,
|
|
show_stats=args.show_stats,
|
|
stats_only=args.stats_only,
|
|
histogram=args.histogram,
|
|
verbose_log=args.verbose_log)
|
|
|
|
|
|
def update(parser, args):
|
|
import SATestUpdateDiffs
|
|
from ProjectMap import ProjectMap
|
|
|
|
project_map = ProjectMap()
|
|
for project in project_map.projects:
|
|
SATestUpdateDiffs.update_reference_results(project, args.git)
|
|
|
|
|
|
def benchmark(parser, args):
|
|
from SATestBenchmark import Benchmark
|
|
|
|
projects = get_projects(parser, args)
|
|
benchmark = Benchmark(projects, args.iterations, args.output)
|
|
benchmark.run()
|
|
|
|
|
|
def benchmark_compare(parser, args):
|
|
import SATestBenchmark
|
|
SATestBenchmark.compare(args.old, args.new, args.output)
|
|
|
|
|
|
def get_projects(parser, args):
|
|
from ProjectMap import ProjectMap, Size
|
|
|
|
project_map = ProjectMap()
|
|
projects = project_map.projects
|
|
|
|
def filter_projects(projects, predicate, force=False):
|
|
return [project.with_fields(enabled=(force or project.enabled) and
|
|
predicate(project))
|
|
for project in projects]
|
|
|
|
if args.projects:
|
|
projects_arg = args.projects.split(",")
|
|
available_projects = [project.name
|
|
for project in projects]
|
|
|
|
# validate that given projects are present in the project map file
|
|
for manual_project in projects_arg:
|
|
if manual_project not in available_projects:
|
|
parser.error("Project '{project}' is not found in "
|
|
"the project map file. Available projects are "
|
|
"{all}.".format(project=manual_project,
|
|
all=available_projects))
|
|
|
|
projects = filter_projects(projects, lambda project:
|
|
project.name in projects_arg,
|
|
force=True)
|
|
|
|
try:
|
|
max_size = Size.from_str(args.max_size)
|
|
except ValueError as e:
|
|
parser.error("{}".format(e))
|
|
|
|
projects = filter_projects(projects, lambda project:
|
|
project.size <= max_size)
|
|
|
|
return projects
|
|
|
|
|
|
def docker(parser, args):
|
|
if len(args.rest) > 0:
|
|
if args.rest[0] != "--":
|
|
parser.error("REST arguments should start with '--'")
|
|
args.rest = args.rest[1:]
|
|
|
|
if args.build_image:
|
|
docker_build_image()
|
|
elif args.shell:
|
|
docker_shell(args)
|
|
else:
|
|
sys.exit(docker_run(args, ' '.join(args.rest)))
|
|
|
|
|
|
def docker_build_image():
|
|
sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR),
|
|
shell=True))
|
|
|
|
|
|
def docker_shell(args):
|
|
try:
|
|
# First we need to start the docker container in a waiting mode,
|
|
# so it doesn't do anything, but most importantly keeps working
|
|
# while the shell session is in progress.
|
|
docker_run(args, "--wait", "--detach")
|
|
# Since the docker container is running, we can actually connect to it
|
|
call("docker exec -it satest bash", shell=True)
|
|
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
finally:
|
|
docker_cleanup()
|
|
|
|
|
|
def docker_run(args, command, docker_args=""):
|
|
try:
|
|
return call("docker run --rm --name satest "
|
|
"-v {llvm}:/llvm-project "
|
|
"-v {build}:/build "
|
|
"-v {clang}:/analyzer "
|
|
"-v {scripts}:/scripts "
|
|
"-v {projects}:/projects "
|
|
"{docker_args} "
|
|
"satest-image:latest {command}"
|
|
.format(llvm=args.llvm_project_dir,
|
|
build=args.build_dir,
|
|
clang=args.clang_dir,
|
|
scripts=SCRIPTS_DIR,
|
|
projects=PROJECTS_DIR,
|
|
docker_args=docker_args,
|
|
command=command),
|
|
shell=True)
|
|
|
|
except KeyboardInterrupt:
|
|
docker_cleanup()
|
|
|
|
|
|
def docker_cleanup():
|
|
print("Please wait for docker to clean up")
|
|
call("docker stop satest", shell=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers()
|
|
|
|
# add subcommand
|
|
add_parser = subparsers.add_parser(
|
|
"add",
|
|
help="Add a new project for the analyzer testing.")
|
|
# TODO: Add an option not to build.
|
|
# TODO: Set the path to the Repository directory.
|
|
add_parser.add_argument("name", nargs=1, help="Name of the new project")
|
|
add_parser.add_argument("--mode", action="store", default=1, type=int,
|
|
choices=[0, 1, 2],
|
|
help="Build mode: 0 for single file project, "
|
|
"1 for scan_build, "
|
|
"2 for single file c++11 project")
|
|
add_parser.add_argument("--source", action="store", default="script",
|
|
choices=["script", "git", "zip"],
|
|
help="Source type of the new project: "
|
|
"'git' for getting from git "
|
|
"(please provide --origin and --commit), "
|
|
"'zip' for unpacking source from a zip file, "
|
|
"'script' for downloading source by running "
|
|
"a custom script")
|
|
add_parser.add_argument("--origin", action="store", default="",
|
|
help="Origin link for a git repository")
|
|
add_parser.add_argument("--commit", action="store", default="",
|
|
help="Git hash for a commit to checkout")
|
|
add_parser.set_defaults(func=add)
|
|
|
|
# build subcommand
|
|
build_parser = subparsers.add_parser(
|
|
"build",
|
|
help="Build projects from the project map and compare results with "
|
|
"the reference.")
|
|
build_parser.add_argument("--strictness", dest="strictness",
|
|
type=int, default=0,
|
|
help="0 to fail on runtime errors, 1 to fail "
|
|
"when the number of found bugs are different "
|
|
"from the reference, 2 to fail on any "
|
|
"difference from the reference. Default is 0.")
|
|
build_parser.add_argument("-r", dest="regenerate", action="store_true",
|
|
default=False,
|
|
help="Regenerate reference output.")
|
|
build_parser.add_argument("--override-compiler", action="store_true",
|
|
default=False, help="Call scan-build with "
|
|
"--override-compiler option.")
|
|
build_parser.add_argument("-j", "--jobs", dest="jobs",
|
|
type=int, default=0,
|
|
help="Number of projects to test concurrently")
|
|
build_parser.add_argument("--extra-analyzer-config",
|
|
dest="extra_analyzer_config", type=str,
|
|
default="",
|
|
help="Arguments passed to to -analyzer-config")
|
|
build_parser.add_argument("--projects", action="store", default="",
|
|
help="Comma-separated list of projects to test")
|
|
build_parser.add_argument("--max-size", action="store", default=None,
|
|
help="Maximum size for the projects to test")
|
|
build_parser.add_argument("-v", "--verbose", action="count", default=0)
|
|
build_parser.set_defaults(func=build)
|
|
|
|
# compare subcommand
|
|
cmp_parser = subparsers.add_parser(
|
|
"compare",
|
|
help="Comparing two static analyzer runs in terms of "
|
|
"reported warnings and execution time statistics.")
|
|
cmp_parser.add_argument("--root-old", dest="root_old",
|
|
help="Prefix to ignore on source files for "
|
|
"OLD directory",
|
|
action="store", type=str, default="")
|
|
cmp_parser.add_argument("--root-new", dest="root_new",
|
|
help="Prefix to ignore on source files for "
|
|
"NEW directory",
|
|
action="store", type=str, default="")
|
|
cmp_parser.add_argument("--verbose-log", dest="verbose_log",
|
|
help="Write additional information to LOG "
|
|
"[default=None]",
|
|
action="store", type=str, default=None,
|
|
metavar="LOG")
|
|
cmp_parser.add_argument("--stats-only", action="store_true",
|
|
dest="stats_only", default=False,
|
|
help="Only show statistics on reports")
|
|
cmp_parser.add_argument("--show-stats", action="store_true",
|
|
dest="show_stats", default=False,
|
|
help="Show change in statistics")
|
|
cmp_parser.add_argument("--histogram", action="store", default=None,
|
|
help="Show histogram of paths differences. "
|
|
"Requires matplotlib")
|
|
cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
|
|
cmp_parser.add_argument("new", nargs=1, help="Directory with new results")
|
|
cmp_parser.set_defaults(func=compare)
|
|
|
|
# update subcommand
|
|
upd_parser = subparsers.add_parser(
|
|
"update",
|
|
help="Update static analyzer reference results based on the previous "
|
|
"run of SATest build. Assumes that SATest build was just run.")
|
|
upd_parser.add_argument("--git", action="store_true",
|
|
help="Stage updated results using git.")
|
|
upd_parser.set_defaults(func=update)
|
|
|
|
# docker subcommand
|
|
dock_parser = subparsers.add_parser(
|
|
"docker",
|
|
help="Run regression system in the docker.")
|
|
|
|
dock_parser.add_argument("--build-image", action="store_true",
|
|
help="Build docker image for running tests.")
|
|
dock_parser.add_argument("--shell", action="store_true",
|
|
help="Start a shell on docker.")
|
|
dock_parser.add_argument("--llvm-project-dir", action="store",
|
|
default=DEFAULT_LLVM_DIR,
|
|
help="Path to LLVM source code. Defaults "
|
|
"to the repo where this script is located. ")
|
|
dock_parser.add_argument("--build-dir", action="store", default="",
|
|
help="Path to a directory where docker should "
|
|
"build LLVM code.")
|
|
dock_parser.add_argument("--clang-dir", action="store", default="",
|
|
help="Path to find/install LLVM installation.")
|
|
dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[],
|
|
help="Additionall args that will be forwarded "
|
|
"to the docker's entrypoint.")
|
|
dock_parser.set_defaults(func=docker)
|
|
|
|
# benchmark subcommand
|
|
bench_parser = subparsers.add_parser(
|
|
"benchmark",
|
|
help="Run benchmarks by building a set of projects multiple times.")
|
|
|
|
bench_parser.add_argument("-i", "--iterations", action="store",
|
|
type=int, default=20,
|
|
help="Number of iterations for building each "
|
|
"project.")
|
|
bench_parser.add_argument("-o", "--output", action="store",
|
|
default="benchmark.csv",
|
|
help="Output csv file for the benchmark results")
|
|
bench_parser.add_argument("--projects", action="store", default="",
|
|
help="Comma-separated list of projects to test")
|
|
bench_parser.add_argument("--max-size", action="store", default=None,
|
|
help="Maximum size for the projects to test")
|
|
bench_parser.set_defaults(func=benchmark)
|
|
|
|
bench_subparsers = bench_parser.add_subparsers()
|
|
bench_compare_parser = bench_subparsers.add_parser(
|
|
"compare",
|
|
help="Compare benchmark runs.")
|
|
bench_compare_parser.add_argument("--old", action="store", required=True,
|
|
help="Benchmark reference results to "
|
|
"compare agains.")
|
|
bench_compare_parser.add_argument("--new", action="store", required=True,
|
|
help="New benchmark results to check.")
|
|
bench_compare_parser.add_argument("-o", "--output",
|
|
action="store", required=True,
|
|
help="Output file for plots.")
|
|
bench_compare_parser.set_defaults(func=benchmark_compare)
|
|
|
|
args = parser.parse_args()
|
|
args.func(parser, args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|