[CI] Enable Build Failure Reporting

This patch finishes up the plumbing so that generate_test_report will dump build
failures into the Github checks summary.

Reviewers: Keenuts, lnihlen, dschuff, gburgessiv, cmtice, DavidSpickett

Reviewed By: cmtice, DavidSpickett

Pull Request: https://github.com/llvm/llvm-project/pull/152622

.ci/generate_test_report_github.py

@@ -16,11 +16,13 @@ PLATFORM_TITLES = {
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("return_code", help="The build's return code.", type=int)
-    parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
+    parser.add_argument(
+        "build_test_logs", help="Paths to JUnit report files and ninja logs.", nargs="*"
+    )
     args = parser.parse_args()
 
     report = generate_test_report_lib.generate_report_from_files(
-        PLATFORM_TITLES[platform.system()], args.return_code, args.junit_files
+        PLATFORM_TITLES[platform.system()], args.return_code, args.build_test_logs
     )
 
     print(report)

.ci/generate_test_report_lib.py

@@ -251,7 +251,19 @@ def generate_report(
     return report
 
 
-def generate_report_from_files(title, return_code, junit_files):
+def generate_report_from_files(title, return_code, build_log_files):
+    junit_files = [
+        junit_file for junit_file in build_log_files if junit_file.endswith(".xml")
+    ]
+    ninja_log_files = [
+        ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log")
+    ]
+    ninja_logs = []
+    for ninja_log_file in ninja_log_files:
+        with open(ninja_log_file, "r") as ninja_log_file_handle:
+            ninja_logs.append(
+                [log_line.strip() for log_line in ninja_log_file_handle.readlines()]
+            )
     return generate_report(
-        title, return_code, [JUnitXml.fromfile(p) for p in junit_files], []
+        title, return_code, [JUnitXml.fromfile(p) for p in junit_files], ninja_logs
     )

.ci/generate_test_report_lib_test.py

@@ -8,6 +8,8 @@
 import unittest
 from io import StringIO
 from textwrap import dedent
+import tempfile
+import os
 
 from junitparser import JUnitXml
 
@@ -718,3 +720,59 @@ class TestReports(unittest.TestCase):
                 )
             ),
         )
+
+    def test_generate_report_end_to_end(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            junit_xml_file = os.path.join(temp_dir, "junit.xml")
+            with open(junit_xml_file, "w") as junit_xml_handle:
+                junit_xml_handle.write(
+                    dedent(
+                        """\
+                        <?xml version="1.0" encoding="UTF-8"?>
+                        <testsuites time="0.00">
+                        <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+                        <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+                        </testsuite>
+                        </testsuites>"""
+                    )
+                )
+            ninja_log_file = os.path.join(temp_dir, "ninja.log")
+            with open(ninja_log_file, "w") as ninja_log_handle:
+                ninja_log_handle.write(
+                    dedent(
+                        """\
+                        [1/5] test/1.stamp
+                        [2/5] test/2.stamp
+                        [3/5] test/3.stamp
+                        [4/5] test/4.stamp
+                        FAILED: test/4.stamp
+                        touch test/4.stamp
+                        Wow! That's so True!
+                        [5/5] test/5.stamp"""
+                    )
+                )
+            self.assertEqual(
+                generate_test_report_lib.generate_report_from_files(
+                    "Foo", 1, [junit_xml_file, ninja_log_file]
+                ),
+                dedent(
+                    """\
+                    # Foo
+
+                    * 1 test passed
+
+                    All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+                    <details>
+                    <summary>test/4.stamp</summary>
+
+                    ```
+                    FAILED: test/4.stamp
+                    touch test/4.stamp
+                    Wow! That's so True!
+                    ```
+                    </details>
+
+                    If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+                ),
+            )

.ci/utils.sh

@@ -33,7 +33,8 @@ function at-exit {
 
   if [[ "$GITHUB_STEP_SUMMARY" != "" ]]; then
     python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py \
-      $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
+      $retcode "${BUILD_DIR}"/test-results.*.xml "${BUILD_DIR}"/ninja*.log \
+      >> $GITHUB_STEP_SUMMARY
   fi
 }
 trap at-exit EXIT