diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py index a1e0406daafe..d34319131ab6 100644 --- a/clang/test/lit.cfg.py +++ b/clang/test/lit.cfg.py @@ -410,3 +410,13 @@ if "system-aix" in config.available_features: # possibly be present in system and user configuration files, so disable # default configs for the test runs. config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1" + +if lit_config.update_tests: + import sys + import os + + utilspath = os.path.join(config.llvm_src_root, "utils") + sys.path.append(utilspath) + from update_any_test_checks import utc_lit_plugin + + lit_config.test_updaters.append(utc_lit_plugin) diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst index eb90e950a377..15c249d8e6d3 100644 --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -399,6 +399,11 @@ ADDITIONAL OPTIONS Show all features used in the test suite (in ``XFAIL``, ``UNSUPPORTED`` and ``REQUIRES``) and exit. +.. option:: --update-tests + + Pass failing tests to functions in the ``lit_config.test_updaters`` list to + check whether any of them know how to update the test to make it pass. + EXIT STATUS ----------- diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py index 8c2d1a454e8f..bc240425d6d0 100644 --- a/llvm/test/lit.cfg.py +++ b/llvm/test/lit.cfg.py @@ -715,3 +715,13 @@ if "system-aix" in config.available_features: if config.has_logf128: config.available_features.add("has_logf128") + +if lit_config.update_tests: + import sys + import os + + utilspath = os.path.join(config.llvm_src_root, "utils") + sys.path.append(utilspath) + from update_any_test_checks import utc_lit_plugin + + lit_config.test_updaters.append(utc_lit_plugin) diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py index cb4aef6f72a8..df297b91be1b 100644 --- a/llvm/utils/lit/lit/LitConfig.py +++ b/llvm/utils/lit/lit/LitConfig.py @@ -39,6 +39,7 @@ class LitConfig(object): parallelism_groups={}, per_test_coverage=False, gtest_sharding=True, + update_tests=False, ): # The name of the test runner. self.progname = progname @@ -91,6 +92,8 @@ class LitConfig(object): self.parallelism_groups = parallelism_groups self.per_test_coverage = per_test_coverage self.gtest_sharding = bool(gtest_sharding) + self.update_tests = update_tests + self.test_updaters = [] @property def maxIndividualTestTime(self): diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py index e7cd70766a3d..f2c5c6d0dbe9 100644 --- a/llvm/utils/lit/lit/TestRunner.py +++ b/llvm/utils/lit/lit/TestRunner.py @@ -1192,6 +1192,18 @@ def executeScriptInternal( str(result.timeoutReached), ) + if litConfig.update_tests: + for test_updater in litConfig.test_updaters: + try: + update_output = test_updater(result, test) + except Exception as e: + out += f"Exception occurred in test updater: {e}" + continue + if update_output: + for line in update_output.splitlines(): + out += f"# {line}\n" + break + return out, err, exitCode, timeoutInfo diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py index e88951520e66..8f9211ee3f53 100644 --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -230,6 +230,12 @@ def parse_args(): action="store_true", help="Exit with status zero even if some tests fail", ) + execution_group.add_argument( + "--update-tests", + dest="update_tests", + action="store_true", + help="Try to update regression tests to reflect current behavior, if possible", + ) execution_test_time_group = execution_group.add_mutually_exclusive_group() execution_test_time_group.add_argument( "--skip-test-time-recording", diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py index b04fb25f3c03..56aa5eb64fa3 100644 --- a/llvm/utils/lit/lit/llvm/config.py +++ b/llvm/utils/lit/lit/llvm/config.py @@ -64,12 +64,17 @@ class LLVMConfig(object): self.with_environment("_TAG_REDIR_ERR", "TXT") self.with_environment("_CEE_RUNOPTS", "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)") + if lit_config.update_tests: + self.use_lit_shell = True + # Choose between lit's internal shell pipeline runner and a real shell. # If LIT_USE_INTERNAL_SHELL is in the environment, we use that as an # override. lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL") if lit_shell_env: self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env) + if not self.use_lit_shell and lit_config.update_tests: + print("note: --update-tests is not supported when using external shell") if not self.use_lit_shell: features.add("shell") diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py index 9650a0e90117..5255e2c5e1b5 100755 --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -43,6 +43,7 @@ def main(builtin_params={}): per_test_coverage=opts.per_test_coverage, gtest_sharding=opts.gtest_sharding, maxRetriesPerTest=opts.maxRetriesPerTest, + update_tests=opts.update_tests, ) discovered_tests = lit.discovery.find_tests_for_inputs( diff --git a/llvm/utils/update_any_test_checks.py b/llvm/utils/update_any_test_checks.py index e8eef1a46c50..76fe33659392 100755 --- a/llvm/utils/update_any_test_checks.py +++ b/llvm/utils/update_any_test_checks.py @@ -34,9 +34,12 @@ def find_utc_tool(search_path, utc_name): return None -def run_utc_tool(utc_name, utc_tool, testname): +def run_utc_tool(utc_name, utc_tool, testname, environment): result = subprocess.run( - [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE + [utc_tool, testname], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, ) return (result.returncode, result.stdout, result.stderr) @@ -60,6 +63,42 @@ def expand_listfile_args(arg_list): return exp_arg_list +def utc_lit_plugin(result, test): + testname = test.getFilePath() + if not testname: + return None + + script_name = os.path.abspath(__file__) + utc_search_path = os.path.join(os.path.dirname(script_name), os.path.pardir) + + with open(testname, "r") as f: + header = f.readline().strip() + + m = RE_ASSERTIONS.search(header) + if m is None: + return None + + utc_name = m.group(1) + utc_tool = find_utc_tool([utc_search_path], utc_name) + if not utc_tool: + return f"update-utc-tests: {utc_name} not found" + + return_code, stdout, stderr = run_utc_tool( + utc_name, utc_tool, testname, test.config.environment + ) + + stderr = stderr.decode(errors="replace") + if return_code != 0: + if stderr: + return f"update-utc-tests: {utc_name} exited with return code {return_code}\n{stderr.rstrip()}" + return f"update-utc-tests: {utc_name} exited with return code {return_code}" + + stdout = stdout.decode(errors="replace") + if stdout: + return f"update-utc-tests: updated {testname}\n{stdout.rstrip()}" + return f"update-utc-tests: updated {testname}" + + def main(): from argparse import RawTextHelpFormatter @@ -78,6 +117,11 @@ def main(): nargs="*", help="Additional directories to scan for update_*_test_checks scripts", ) + parser.add_argument( + "--path", + help="""Additional directories to scan for executables invoked by the update_*_test_checks scripts, +separated by the platform path separator""", + ) parser.add_argument("tests", nargs="+") config = parser.parse_args() @@ -88,6 +132,10 @@ def main(): script_name = os.path.abspath(__file__) utc_search_path.append(os.path.join(os.path.dirname(script_name), os.path.pardir)) + local_env = os.environ.copy() + if config.path: + local_env["PATH"] = config.path + os.pathsep + local_env["PATH"] + not_autogenerated = [] utc_tools = {} have_error = False @@ -117,7 +165,7 @@ def main(): continue future = executor.submit( - run_utc_tool, utc_name, utc_tools[utc_name], testname + run_utc_tool, utc_name, utc_tools[utc_name], testname, local_env ) jobs.append((testname, future))