e6cc7b723f
The issue with these test failures is that the dSYM was not being found by lldb, which is why setting breakpoints was failing and lldb quit without performing any steps. This change copies the dSYM to the same temp directory that the executable is copied to.
279 lines
10 KiB
Python
279 lines
10 KiB
Python
# DExTer : Debugging Experience Tester
|
|
# ~~~~~~ ~ ~~ ~ ~~
|
|
#
|
|
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
# See https://llvm.org/LICENSE.txt for license information.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
"""Test tool."""
|
|
|
|
import math
|
|
import os
|
|
import csv
|
|
import pickle
|
|
import shutil
|
|
import platform
|
|
|
|
from dex.command.ParseCommand import get_command_infos
|
|
from dex.debugger.Debuggers import run_debugger_subprocess
|
|
from dex.debugger.DebuggerControllers.DefaultController import DefaultController
|
|
from dex.debugger.DebuggerControllers.ConditionalController import ConditionalController
|
|
from dex.dextIR.DextIR import DextIR
|
|
from dex.heuristic import Heuristic
|
|
from dex.tools import TestToolBase
|
|
from dex.utils.Exceptions import DebuggerException
|
|
from dex.utils.Exceptions import BuildScriptException, HeuristicException
|
|
from dex.utils.PrettyOutputBase import Stream
|
|
from dex.utils.ReturnCode import ReturnCode
|
|
|
|
|
|
class TestCase(object):
|
|
def __init__(self, context, name, heuristic, error):
|
|
self.context = context
|
|
self.name = name
|
|
self.heuristic = heuristic
|
|
self.error = error
|
|
|
|
@property
|
|
def penalty(self):
|
|
try:
|
|
return self.heuristic.penalty
|
|
except AttributeError:
|
|
return float("nan")
|
|
|
|
@property
|
|
def max_penalty(self):
|
|
try:
|
|
return self.heuristic.max_penalty
|
|
except AttributeError:
|
|
return float("nan")
|
|
|
|
@property
|
|
def score(self):
|
|
try:
|
|
return self.heuristic.score
|
|
except AttributeError:
|
|
return float("nan")
|
|
|
|
def __str__(self):
|
|
if self.error and self.context.options.verbose:
|
|
verbose_error = str(self.error)
|
|
else:
|
|
verbose_error = ""
|
|
|
|
if self.error:
|
|
script_error = (
|
|
" : {}".format(self.error.script_error.splitlines()[0])
|
|
if getattr(self.error, "script_error", None)
|
|
else ""
|
|
)
|
|
|
|
error = " [{}{}]".format(str(self.error).splitlines()[0], script_error)
|
|
else:
|
|
error = ""
|
|
|
|
try:
|
|
summary = self.heuristic.summary_string
|
|
except AttributeError:
|
|
summary = "<r>nan/nan (nan)</>"
|
|
return "{}: {}{}\n{}".format(self.name, summary, error, verbose_error)
|
|
|
|
|
|
class Tool(TestToolBase):
|
|
"""Run the specified DExTer test(s) with the specified compiler and linker
|
|
options and produce a dextIR file as well as printing out the debugging
|
|
experience score calculated by the DExTer heuristic.
|
|
"""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super(Tool, self).__init__(*args, **kwargs)
|
|
self._test_cases = []
|
|
|
|
@property
|
|
def name(self):
|
|
return "DExTer test"
|
|
|
|
def add_tool_arguments(self, parser, defaults):
|
|
parser.add_argument(
|
|
"--fail-lt",
|
|
type=float,
|
|
default=0.0, # By default TEST always succeeds.
|
|
help="exit with status FAIL(2) if the test result"
|
|
" is less than this value.",
|
|
metavar="<float>",
|
|
)
|
|
parser.add_argument(
|
|
"--calculate-average",
|
|
action="store_true",
|
|
help="calculate the average score of every test run",
|
|
)
|
|
super(Tool, self).add_tool_arguments(parser, defaults)
|
|
|
|
def _init_debugger_controller(self):
|
|
step_collection = DextIR(
|
|
executable_path=self.context.options.executable,
|
|
source_paths=self.context.options.source_files,
|
|
dexter_version=self.context.version,
|
|
)
|
|
|
|
step_collection.commands, new_source_files = get_command_infos(
|
|
self.context.options.test_files, self.context.options.source_root_dir
|
|
)
|
|
|
|
self.context.options.source_files.extend(list(new_source_files))
|
|
|
|
if "DexLimitSteps" in step_collection.commands:
|
|
debugger_controller = ConditionalController(self.context, step_collection)
|
|
else:
|
|
debugger_controller = DefaultController(self.context, step_collection)
|
|
|
|
return debugger_controller
|
|
|
|
def _get_steps(self):
|
|
"""Generate a list of debugger steps from a test case."""
|
|
debugger_controller = self._init_debugger_controller()
|
|
debugger_controller = run_debugger_subprocess(
|
|
debugger_controller, self.context.working_directory.path
|
|
)
|
|
steps = debugger_controller.step_collection
|
|
return steps
|
|
|
|
def _get_results_basename(self, test_name):
|
|
def splitall(x):
|
|
while len(x) > 0:
|
|
x, y = os.path.split(x)
|
|
yield y
|
|
|
|
all_components = reversed([x for x in splitall(test_name)])
|
|
return "_".join(all_components)
|
|
|
|
def _get_results_path(self, test_name):
|
|
"""Returns the path to the test results directory for the test denoted
|
|
by test_name.
|
|
"""
|
|
assert self.context.options.results_directory != None
|
|
return os.path.join(
|
|
self.context.options.results_directory,
|
|
self._get_results_basename(test_name),
|
|
)
|
|
|
|
def _get_results_text_path(self, test_name):
|
|
"""Returns path results .txt file for test denoted by test_name."""
|
|
test_results_path = self._get_results_path(test_name)
|
|
return "{}.txt".format(test_results_path)
|
|
|
|
def _get_results_pickle_path(self, test_name):
|
|
"""Returns path results .dextIR file for test denoted by test_name."""
|
|
test_results_path = self._get_results_path(test_name)
|
|
return "{}.dextIR".format(test_results_path)
|
|
|
|
def _record_steps(self, test_name, steps):
|
|
"""Write out the set of steps out to the test's .txt and .json
|
|
results file if a results directory has been specified.
|
|
"""
|
|
if self.context.options.results_directory:
|
|
output_text_path = self._get_results_text_path(test_name)
|
|
with open(output_text_path, "w") as fp:
|
|
self.context.o.auto(str(steps), stream=Stream(fp))
|
|
|
|
output_dextIR_path = self._get_results_pickle_path(test_name)
|
|
with open(output_dextIR_path, "wb") as fp:
|
|
pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL)
|
|
|
|
def _record_score(self, test_name, heuristic):
|
|
"""Write out the test's heuristic score to the results .txt file
|
|
if a results directory has been specified.
|
|
"""
|
|
if self.context.options.results_directory:
|
|
output_text_path = self._get_results_text_path(test_name)
|
|
with open(output_text_path, "a") as fp:
|
|
self.context.o.auto(heuristic.verbose_output, stream=Stream(fp))
|
|
|
|
def _record_test_and_display(self, test_case):
|
|
"""Output test case to o stream and record test case internally for
|
|
handling later.
|
|
"""
|
|
self.context.o.auto(test_case)
|
|
self._test_cases.append(test_case)
|
|
|
|
def _record_failed_test(self, test_name, exception):
|
|
"""Instantiate a failed test case with failure exception and
|
|
store internally.
|
|
"""
|
|
test_case = TestCase(self.context, test_name, None, exception)
|
|
self._record_test_and_display(test_case)
|
|
|
|
def _record_successful_test(self, test_name, steps, heuristic):
|
|
"""Instantiate a successful test run, store test for handling later.
|
|
Display verbose output for test case if required.
|
|
"""
|
|
test_case = TestCase(self.context, test_name, heuristic, None)
|
|
self._record_test_and_display(test_case)
|
|
if self.context.options.verbose:
|
|
self.context.o.auto("\n{}\n".format(steps))
|
|
self.context.o.auto(heuristic.verbose_output)
|
|
|
|
def _run_test(self, test_name):
|
|
"""Attempt to run test files specified in options.source_files. Store
|
|
result internally in self._test_cases.
|
|
"""
|
|
try:
|
|
if self.context.options.binary:
|
|
if platform.system() == 'Darwin' and os.path.exists(self.context.options.binary + '.dSYM'):
|
|
# On Darwin, the debug info is in the .dSYM which might not be found by lldb, copy it into the tmp working directory
|
|
shutil.copytree(self.context.options.binary + '.dSYM', self.context.options.executable + '.dSYM')
|
|
# Copy user's binary into the tmp working directory.
|
|
shutil.copy(
|
|
self.context.options.binary, self.context.options.executable
|
|
)
|
|
steps = self._get_steps()
|
|
self._record_steps(test_name, steps)
|
|
heuristic_score = Heuristic(self.context, steps)
|
|
self._record_score(test_name, heuristic_score)
|
|
except (BuildScriptException, DebuggerException, HeuristicException) as e:
|
|
self._record_failed_test(test_name, e)
|
|
return
|
|
|
|
self._record_successful_test(test_name, steps, heuristic_score)
|
|
return
|
|
|
|
def _handle_results(self) -> ReturnCode:
|
|
return_code = ReturnCode.OK
|
|
options = self.context.options
|
|
|
|
if not options.verbose:
|
|
self.context.o.auto("\n")
|
|
|
|
if options.calculate_average:
|
|
# Calculate and print the average score
|
|
score_sum = 0.0
|
|
num_tests = 0
|
|
for test_case in self._test_cases:
|
|
score = test_case.score
|
|
if not test_case.error and not math.isnan(score):
|
|
score_sum += test_case.score
|
|
num_tests += 1
|
|
|
|
if num_tests != 0:
|
|
print("@avg: ({:.4f})".format(score_sum / num_tests))
|
|
|
|
has_failed = lambda test: test.score < options.fail_lt or test.error
|
|
if any(map(has_failed, self._test_cases)):
|
|
return_code = ReturnCode.FAIL
|
|
|
|
if options.results_directory:
|
|
summary_path = os.path.join(options.results_directory, "summary.csv")
|
|
with open(summary_path, mode="w", newline="") as fp:
|
|
writer = csv.writer(fp, delimiter=",")
|
|
writer.writerow(["Test Case", "Score", "Error"])
|
|
|
|
for test_case in self._test_cases:
|
|
writer.writerow(
|
|
[
|
|
test_case.name,
|
|
"{:.4f}".format(test_case.score),
|
|
test_case.error,
|
|
]
|
|
)
|
|
|
|
return return_code
|