From 498aeada7b8e079f5bc59bf4e9bb0c8630ee6d32 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Thu, 10 Jul 2025 06:41:37 -0700
Subject: [PATCH] [CI] Generate Test Report With No Test Results

This patch makes it so that generate_test_report_github.py generates a
test report even when we don't get any test results. This otherwise
created a pretty confusing user experience on the Github side if the
build failed before any tests ran or in cases like running check-libc
where none of the tests are run through lit.

Reviewers: lnihlen, cmtice

Pull Request: https://github.com/llvm/llvm-project/pull/147871
---
 .ci/generate_test_report_lib.py      | 53 ++++++++++++++++------------
 .ci/generate_test_report_lib_test.py | 36 +++++++++++++++++--
 2 files changed, 65 insertions(+), 24 deletions(-)

diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index dd203b5169d9..25d810f1c6d1 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -5,6 +5,14 @@
 
 from junitparser import JUnitXml, Failure
 
+SEE_BUILD_FILE_STR = "Download the build's log file to see the details."
+UNRELATED_FAILURES_STR = (
+    "If these failures are unrelated to your changes (for example "
+    "tests are broken or flaky at HEAD), please open an issue at "
+    "https://github.com/llvm/llvm-project/issues and add the "
+    "`infrastructure` label."
+)
+
 
 # Set size_limit to limit the byte size of the report. The default is 1MB as this
 # is the most that can be put into an annotation. If the generated report exceeds
@@ -19,14 +27,6 @@ def generate_report(
     size_limit=1024 * 1024,
     list_failures=True,
 ):
-    if not junit_objects:
-        # Note that we do not post an empty report, therefore we can ignore a
-        # non-zero return code in situations like this.
-        #
-        # If we were going to post a report, then yes, it would be misleading
-        # to say we succeeded when the final return code was non-zero.
-        return ""
-
     failures = {}
     tests_run = 0
     tests_skipped = 0
@@ -50,11 +50,28 @@ def generate_report(
                         (test.classname + "/" + test.name, test.result[0].text)
                     )
 
-    if not tests_run:
-        return ""
-
     report = [f"# {title}", ""]
 
+    if tests_run == 0:
+        if return_code == 0:
+            report.extend(
+                [
+                    "The build succeeded and no tests ran. This is expected in some "
+                    "build configurations."
+                ]
+            )
+        else:
+            report.extend(
+                [
+                    "The build failed before running any tests.",
+                    "",
+                    SEE_BUILD_FILE_STR,
+                    "",
+                    UNRELATED_FAILURES_STR,
+                ]
+            )
+        return "\n".join(report)
+
     tests_passed = tests_run - tests_skipped - tests_failed
 
     def plural(num_tests):
@@ -72,7 +89,7 @@ def generate_report(
             [
                 "",
                 "Failed tests and their output was too large to report. "
-                "Download the build's log file to see the details.",
+                + SEE_BUILD_FILE_STR,
             ]
         )
     elif failures:
@@ -102,20 +119,12 @@ def generate_report(
                 "",
                 "All tests passed but another part of the build **failed**.",
                 "",
-                "Download the build's log file to see the details.",
+                SEE_BUILD_FILE_STR,
             ]
         )
 
     if failures or return_code != 0:
-        report.extend(
-            [
-                "",
-                "If these failures are unrelated to your changes (for example "
-                "tests are broken or flaky at HEAD), please open an issue at "
-                "https://github.com/llvm/llvm-project/issues and add the "
-                "`infrastructure` label.",
-            ]
-        )
+        report.extend(["", UNRELATED_FAILURES_STR])
 
     report = "\n".join(report)
     if len(report.encode("utf-8")) > size_limit:
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 89dfed97a359..eda76ead19b9 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -20,7 +20,30 @@ def junit_from_xml(xml):
 
 class TestReports(unittest.TestCase):
     def test_title_only(self):
-        self.assertEqual(generate_test_report_lib.generate_report("Foo", 0, []), "")
+        self.assertEqual(
+            generate_test_report_lib.generate_report("Foo", 0, []),
+            dedent(
+                """\
+                # Foo
+
+                The build succeeded and no tests ran. This is expected in some build configurations."""
+            ),
+        )
+
+    def test_title_only_failure(self):
+        self.assertEqual(
+            generate_test_report_lib.generate_report("Foo", 1, []),
+            dedent(
+                """\
+            # Foo
+
+            The build failed before running any tests.
+
+            Download the build's log file to see the details.
+
+            If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+            ),
+        )
 
     def test_no_tests_in_testsuite(self):
         self.assertEqual(
@@ -40,7 +63,16 @@ class TestReports(unittest.TestCase):
                     )
                 ],
             ),
-            "",
+            dedent(
+                """\
+                # Foo
+
+                The build failed before running any tests.
+
+                Download the build's log file to see the details.
+
+                If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+            ),
         )
 
     def test_no_failures(self):