Reland "[Utils] Add new --update-tests flag to llvm-lit" (#153821)

This reverts commit
e495231238
to reland
the --update-tests feature, originally landed in
https://github.com/llvm/llvm-project/pull/108425.

clang/test/lit.cfg.py

@@ -410,3 +410,13 @@ if "system-aix" in config.available_features:
 # possibly be present in system and user configuration files, so disable
 # default configs for the test runs.
 config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1"
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    utilspath = os.path.join(config.llvm_src_root, "utils")
+    sys.path.append(utilspath)
+    from update_any_test_checks import utc_lit_plugin
+
+    lit_config.test_updaters.append(utc_lit_plugin)

llvm/docs/CommandGuide/lit.rst

@@ -399,6 +399,11 @@ ADDITIONAL OPTIONS
  Show all features used in the test suite (in ``XFAIL``, ``UNSUPPORTED`` and
  ``REQUIRES``) and exit.
 
+.. option:: --update-tests
+
+ Pass failing tests to functions in the ``lit_config.test_updaters`` list to
+ check whether any of them know how to update the test to make it pass.
+
 EXIT STATUS
 -----------
 

llvm/test/lit.cfg.py

@@ -715,3 +715,13 @@ if "system-aix" in config.available_features:
 
 if config.has_logf128:
     config.available_features.add("has_logf128")
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    utilspath = os.path.join(config.llvm_src_root, "utils")
+    sys.path.append(utilspath)
+    from update_any_test_checks import utc_lit_plugin
+
+    lit_config.test_updaters.append(utc_lit_plugin)

llvm/utils/lit/lit/LitConfig.py

@@ -39,6 +39,7 @@ class LitConfig(object):
         parallelism_groups={},
         per_test_coverage=False,
         gtest_sharding=True,
+        update_tests=False,
     ):
         # The name of the test runner.
         self.progname = progname
@@ -91,6 +92,8 @@ class LitConfig(object):
         self.parallelism_groups = parallelism_groups
         self.per_test_coverage = per_test_coverage
         self.gtest_sharding = bool(gtest_sharding)
+        self.update_tests = update_tests
+        self.test_updaters = []
 
     @property
     def maxIndividualTestTime(self):

llvm/utils/lit/lit/TestRunner.py

@@ -1192,6 +1192,18 @@ def executeScriptInternal(
                 str(result.timeoutReached),
             )
 
+        if litConfig.update_tests:
+            for test_updater in litConfig.test_updaters:
+                try:
+                    update_output = test_updater(result, test)
+                except Exception as e:
+                    out += f"Exception occurred in test updater: {e}"
+                    continue
+                if update_output:
+                    for line in update_output.splitlines():
+                        out += f"# {line}\n"
+                    break
+
     return out, err, exitCode, timeoutInfo
 
 

llvm/utils/lit/lit/cl_arguments.py

@@ -230,6 +230,12 @@ def parse_args():
         action="store_true",
         help="Exit with status zero even if some tests fail",
     )
+    execution_group.add_argument(
+        "--update-tests",
+        dest="update_tests",
+        action="store_true",
+        help="Try to update regression tests to reflect current behavior, if possible",
+    )
     execution_test_time_group = execution_group.add_mutually_exclusive_group()
     execution_test_time_group.add_argument(
         "--skip-test-time-recording",

llvm/utils/lit/lit/llvm/config.py

@@ -64,12 +64,17 @@ class LLVMConfig(object):
             self.with_environment("_TAG_REDIR_ERR", "TXT")
             self.with_environment("_CEE_RUNOPTS", "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)")
 
+        if lit_config.update_tests:
+            self.use_lit_shell = True
+
         # Choose between lit's internal shell pipeline runner and a real shell.
         # If LIT_USE_INTERNAL_SHELL is in the environment, we use that as an
         # override.
         lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL")
         if lit_shell_env:
             self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
+            if not self.use_lit_shell and lit_config.update_tests:
+                print("note: --update-tests is not supported when using external shell")
 
         if not self.use_lit_shell:
             features.add("shell")

llvm/utils/lit/lit/main.py

@@ -43,6 +43,7 @@ def main(builtin_params={}):
         per_test_coverage=opts.per_test_coverage,
         gtest_sharding=opts.gtest_sharding,
         maxRetriesPerTest=opts.maxRetriesPerTest,
+        update_tests=opts.update_tests,
     )
 
     discovered_tests = lit.discovery.find_tests_for_inputs(

llvm/utils/update_any_test_checks.py

@@ -34,9 +34,12 @@ def find_utc_tool(search_path, utc_name):
     return None
 
 
-def run_utc_tool(utc_name, utc_tool, testname):
+def run_utc_tool(utc_name, utc_tool, testname, environment):
     result = subprocess.run(
-        [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        [utc_tool, testname],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=environment,
     )
     return (result.returncode, result.stdout, result.stderr)
 
@@ -60,6 +63,42 @@ def expand_listfile_args(arg_list):
     return exp_arg_list
 
 
+def utc_lit_plugin(result, test):
+    testname = test.getFilePath()
+    if not testname:
+        return None
+
+    script_name = os.path.abspath(__file__)
+    utc_search_path = os.path.join(os.path.dirname(script_name), os.path.pardir)
+
+    with open(testname, "r") as f:
+        header = f.readline().strip()
+
+    m = RE_ASSERTIONS.search(header)
+    if m is None:
+        return None
+
+    utc_name = m.group(1)
+    utc_tool = find_utc_tool([utc_search_path], utc_name)
+    if not utc_tool:
+        return f"update-utc-tests: {utc_name} not found"
+
+    return_code, stdout, stderr = run_utc_tool(
+        utc_name, utc_tool, testname, test.config.environment
+    )
+
+    stderr = stderr.decode(errors="replace")
+    if return_code != 0:
+        if stderr:
+            return f"update-utc-tests: {utc_name} exited with return code {return_code}\n{stderr.rstrip()}"
+        return f"update-utc-tests: {utc_name} exited with return code {return_code}"
+
+    stdout = stdout.decode(errors="replace")
+    if stdout:
+        return f"update-utc-tests: updated {testname}\n{stdout.rstrip()}"
+    return f"update-utc-tests: updated {testname}"
+
+
 def main():
     from argparse import RawTextHelpFormatter
 
@@ -78,6 +117,11 @@ def main():
         nargs="*",
         help="Additional directories to scan for update_*_test_checks scripts",
     )
+    parser.add_argument(
+        "--path",
+        help="""Additional directories to scan for executables invoked by the update_*_test_checks scripts,
+separated by the platform path separator""",
+    )
     parser.add_argument("tests", nargs="+")
     config = parser.parse_args()
 
@@ -88,6 +132,10 @@ def main():
     script_name = os.path.abspath(__file__)
     utc_search_path.append(os.path.join(os.path.dirname(script_name), os.path.pardir))
 
+    local_env = os.environ.copy()
+    if config.path:
+        local_env["PATH"] = config.path + os.pathsep + local_env["PATH"]
+
     not_autogenerated = []
     utc_tools = {}
     have_error = False
@@ -117,7 +165,7 @@ def main():
                         continue
 
                 future = executor.submit(
-                    run_utc_tool, utc_name, utc_tools[utc_name], testname
+                    run_utc_tool, utc_name, utc_tools[utc_name], testname, local_env
                 )
                 jobs.append((testname, future))